From 98c522bf46b3d0dcb94914a6edd40a2758afbc83 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Thu, 23 Apr 2026 14:50:13 +0530 Subject: [PATCH 01/12] Add support for merging `ReleaseOverride` into Helm values YAML during manifest generation **Details**: - Introduced `mergeReleaseOverrideIntoValuesYaml` method to handle JSON patching of `ReleaseOverride` into values YAML. - Updated `DeploymentTemplateServiceImpl` to integrate the new merge logic. - Added `ReleaseOverride` field to chart beans and updated adaptors for consistent data handling. - Resolved import ordering issues in multiple files. --- pkg/chart/adaptor/adaptor.go | 4 ++- pkg/chart/bean/bean.go | 2 ++ .../DeploymentTemplateService.go | 11 +++++--- pkg/generateManifest/helper.go | 26 +++++++++++++++++++ 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/pkg/chart/adaptor/adaptor.go b/pkg/chart/adaptor/adaptor.go index 8354ee1601..eaa39753ed 100644 --- a/pkg/chart/adaptor/adaptor.go +++ b/pkg/chart/adaptor/adaptor.go @@ -3,13 +3,14 @@ package adaptor import ( "encoding/json" "fmt" + "strings" + apiGitOpsBean "github.com/devtron-labs/devtron/api/bean/gitOps" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/chart/bean" chartRepoRepository "github.com/devtron-labs/devtron/pkg/chartRepo/repository" bean2 "github.com/devtron-labs/devtron/pkg/deployment/common/bean" util2 "github.com/devtron-labs/devtron/util" - "strings" ) // ChartAdaptor converts db object chartRepoRepository.Chart to bean.TemplateRequest @@ -40,6 +41,7 @@ func ChartAdaptor(chartModel *chartRepoRepository.Chart, IsCustomGitRepository: deploymentConfig.ConfigType == bean2.CUSTOM.String(), ImageDescriptorTemplate: chartModel.ImageDescriptorTemplate, TargetRevision: targetRevision, + ReleaseOverride: chartModel.ReleaseOverride, } if chartModel.Latest { templateRequest.LatestChartVersion = chartModel.ChartVersion diff --git a/pkg/chart/bean/bean.go b/pkg/chart/bean/bean.go index 0eb09655ab..7342556534 100644 --- a/pkg/chart/bean/bean.go +++ b/pkg/chart/bean/bean.go @@ -18,6 +18,7 @@ package bean import ( "encoding/json" + "github.com/devtron-labs/devtron/internal/sql/models" bean2 "github.com/devtron-labs/devtron/pkg/pipeline/bean" ) @@ -50,6 +51,7 @@ type TemplateRequest struct { UserId int32 `json:"-"` LatestChartVersion string `json:"-"` ImageDescriptorTemplate string `json:"-"` + ReleaseOverride string `json:"-"` } type ChartUpgradeRequest struct { diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index cb986b7261..b5972facb6 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -19,6 +19,11 @@ package generateManifest import ( "context" "fmt" + "net/http" + "regexp" + "strconv" + "sync" + "github.com/caarlos0/env" "github.com/devtron-labs/common-lib/utils/k8s" "github.com/devtron-labs/devtron/api/helm-app/gRPC" @@ -53,11 +58,7 @@ import ( "golang.org/x/exp/maps" chart2 "helm.sh/helm/v3/pkg/chart" "k8s.io/utils/pointer" - "net/http" - "regexp" "sigs.k8s.io/yaml" - "strconv" - "sync" ) // TODO: Prakash, move this interface to pkg/deployment/manifest/deploymentTemplate, both are same @@ -566,6 +567,8 @@ func (impl DeploymentTemplateServiceImpl) patchReleaseAttributes(request *Deploy mergedValuesYaml = string(mergedYamlBytes) + mergedValuesYaml = impl.mergeReleaseOverrideIntoValuesYaml(mergedValuesYaml, chartDto.ReleaseOverride) + return mergedValuesYaml } diff --git a/pkg/generateManifest/helper.go b/pkg/generateManifest/helper.go index 6f53c12a90..40b658eb1c 100644 --- a/pkg/generateManifest/helper.go +++ b/pkg/generateManifest/helper.go @@ -27,8 +27,34 @@ import ( "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" "go.opentelemetry.io/otel" "golang.org/x/exp/maps" + k8sYaml "sigs.k8s.io/yaml" ) +// mergeReleaseOverrideIntoValuesYaml merges the chart's ReleaseOverride JSON +// on top of the given values YAML. On any failure the input valuesYaml is +// returned unchanged so template rendering can still proceed. +func (impl DeploymentTemplateServiceImpl) mergeReleaseOverrideIntoValuesYaml(valuesYaml, releaseOverrideJson string) string { + if len(releaseOverrideJson) == 0 || releaseOverrideJson == "{}" { + return valuesYaml + } + valuesJsonByte, err := k8sYaml.YAMLToJSON([]byte(valuesYaml)) + if err != nil { + impl.Logger.Errorw("error in converting values yaml to json", "err", err) + return valuesYaml + } + mergedJsonBytes, err := impl.mergeUtil.JsonPatch(valuesJsonByte, []byte(releaseOverrideJson)) + if err != nil { + impl.Logger.Errorw("error in merging releaseOverride into values yaml", "releaseOverrideJson", releaseOverrideJson, "err", err) + return valuesYaml + } + mergedYamlBytes, err := k8sYaml.JSONToYAML(mergedJsonBytes) + if err != nil { + impl.Logger.Errorw("error in converting merged json to yaml", "err", err) + return valuesYaml + } + return string(mergedYamlBytes) +} + func (impl DeploymentTemplateServiceImpl) constructRotatePodResponse(templateChartResponse []*gRPC.TemplateChartResponse, appNameToId map[string]int, environment *repository.Environment) (*RestartPodResponse, error) { appIdToResourceIdentifier := make(map[int]*ResourceIdentifierResponse) for _, tcResp := range templateChartResponse { From 56c0d41662efcfb25dab21723e49818538ce42a5 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Fri, 24 Apr 2026 17:50:35 +0530 Subject: [PATCH 02/12] Support reference chart-based manifest generation before saving the app chart **Details**: - Modified `patchReleaseAttributes` to read `release-values.yaml` and `.image_descriptor_template.json` directly from the reference chart directory. - Added helper methods to handle file discovery and fallback logic for reference chart overrides. - Updated `resolveImageDescriptorTemplate` to handle missing DB entries using reference chart files. - Removed unused `ReleaseOverride` field from chart beans and adaptors. --- pkg/chart/adaptor/adaptor.go | 1 - pkg/chart/bean/bean.go | 1 - .../DeploymentTemplateService.go | 15 +-- pkg/generateManifest/helper.go | 95 +++++++++++++++++++ 4 files changed, 104 insertions(+), 8 deletions(-) diff --git a/pkg/chart/adaptor/adaptor.go b/pkg/chart/adaptor/adaptor.go index eaa39753ed..6e2a6f482f 100644 --- a/pkg/chart/adaptor/adaptor.go +++ b/pkg/chart/adaptor/adaptor.go @@ -41,7 +41,6 @@ func ChartAdaptor(chartModel *chartRepoRepository.Chart, IsCustomGitRepository: deploymentConfig.ConfigType == bean2.CUSTOM.String(), ImageDescriptorTemplate: chartModel.ImageDescriptorTemplate, TargetRevision: targetRevision, - ReleaseOverride: chartModel.ReleaseOverride, } if chartModel.Latest { templateRequest.LatestChartVersion = chartModel.ChartVersion diff --git a/pkg/chart/bean/bean.go b/pkg/chart/bean/bean.go index 7342556534..716b95c7f8 100644 --- a/pkg/chart/bean/bean.go +++ b/pkg/chart/bean/bean.go @@ -51,7 +51,6 @@ type TemplateRequest struct { UserId int32 `json:"-"` LatestChartVersion string `json:"-"` ImageDescriptorTemplate string `json:"-"` - ReleaseOverride string `json:"-"` } type ChartUpgradeRequest struct { diff --git a/pkg/generateManifest/DeploymentTemplateService.go b/pkg/generateManifest/DeploymentTemplateService.go index b5972facb6..618b3a9041 100644 --- a/pkg/generateManifest/DeploymentTemplateService.go +++ b/pkg/generateManifest/DeploymentTemplateService.go @@ -489,7 +489,7 @@ func (impl DeploymentTemplateServiceImpl) GenerateManifest(ctx context.Context, if len(request.DeploymentAppName) != 0 { releaseName = request.DeploymentAppName } - mergedValuesYaml := impl.patchReleaseAttributes(request, valuesYaml) + mergedValuesYaml := impl.patchReleaseAttributes(request, valuesYaml, refChartPath) installReleaseRequest := &gRPC.InstallReleaseRequest{ AppName: request.AppName, ChartName: refChart.Name, @@ -531,7 +531,7 @@ func (impl DeploymentTemplateServiceImpl) GenerateManifest(ctx context.Context, return response, nil } -func (impl DeploymentTemplateServiceImpl) patchReleaseAttributes(request *DeploymentTemplateRequest, valuesYaml string) (mergedValuesYaml string) { +func (impl DeploymentTemplateServiceImpl) patchReleaseAttributes(request *DeploymentTemplateRequest, valuesYaml string, refChartPath string) (mergedValuesYaml string) { mergedValuesYaml = valuesYaml valuesJsonByte, err := yaml.YAMLToJSON([]byte(valuesYaml)) @@ -540,14 +540,13 @@ func (impl DeploymentTemplateServiceImpl) patchReleaseAttributes(request *Deploy return } - chartDto, err := impl.chartReadService.GetByAppIdAndChartRefId(request.AppId, request.ChartRefId) + imageDescriptorTemplate, err := impl.resolveImageDescriptorTemplate(request.AppId, request.ChartRefId, refChartPath) if err != nil { - impl.Logger.Errorw("error in getting the chart using appId and chartRefId", "appId", request.AppId, "chartRefId", request.ChartRefId, "err", err) return } releaseAttributeJson, err := app.NewReleaseAttributes("", "", request.PipelineName, "", - request.AppId, request.EnvId, 0, pointer.Bool(false)).RenderJson(chartDto.ImageDescriptorTemplate) + request.AppId, request.EnvId, 0, pointer.Bool(false)).RenderJson(imageDescriptorTemplate) if err != nil { impl.Logger.Errorw("error in rendering release attributes into image descriptor template ", "releaseAttributeJson", releaseAttributeJson, "err", err) @@ -567,7 +566,11 @@ func (impl DeploymentTemplateServiceImpl) patchReleaseAttributes(request *Deploy mergedValuesYaml = string(mergedYamlBytes) - mergedValuesYaml = impl.mergeReleaseOverrideIntoValuesYaml(mergedValuesYaml, chartDto.ReleaseOverride) + // Read release-values.yaml straight from the reference chart dir so the manifest + // can be rendered before an app-level chart is saved (chartDto.ReleaseOverride is + // only populated post-save). + releaseOverrideJson := impl.readReleaseValuesJsonFromRefChart(refChartPath) + mergedValuesYaml = impl.mergeReleaseOverrideIntoValuesYaml(mergedValuesYaml, releaseOverrideJson) return mergedValuesYaml } diff --git a/pkg/generateManifest/helper.go b/pkg/generateManifest/helper.go index 40b658eb1c..7d2e71ba9d 100644 --- a/pkg/generateManifest/helper.go +++ b/pkg/generateManifest/helper.go @@ -18,18 +18,113 @@ package generateManifest import ( "context" + "errors" "fmt" + "os" + "path/filepath" + "strings" + "github.com/devtron-labs/common-lib/utils/yaml" "github.com/devtron-labs/devtron/api/helm-app/gRPC" "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" clusterBean "github.com/devtron-labs/devtron/pkg/cluster/bean" "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + "github.com/go-pg/pg" "go.opentelemetry.io/otel" "golang.org/x/exp/maps" k8sYaml "sigs.k8s.io/yaml" ) +const ( + releaseValuesYamlFile = "release-values.yaml" + releaseValuesYmlFile = "release-values.yml" + imageDescriptorTemplateFile = ".image_descriptor_template.json" +) + +// readReleaseValuesJsonFromRefChart looks for release-values.yaml / release-values.yml +// at the root of the reference chart directory and returns its content converted to +// JSON. Returns an empty string (without erroring the caller) when the file is absent +// or unreadable — not every chart defines release overrides, and manifest generation +// must still proceed. This mirrors the file-discovery logic in +// ChartTemplateServiceImpl.getValues so behaviour matches the save-time path. +func (impl DeploymentTemplateServiceImpl) readReleaseValuesJsonFromRefChart(refChartPath string) string { + data, ok := impl.readFileFromRefChart(refChartPath, releaseValuesYamlFile, releaseValuesYmlFile) + if !ok { + return "" + } + jsonBytes, err := k8sYaml.YAMLToJSON(data) + if err != nil { + impl.Logger.Errorw("error in converting release-values yaml to json", "refChartPath", refChartPath, "err", err) + return "" + } + return string(jsonBytes) +} + +// readImageDescriptorTemplateFromRefChart reads .image_descriptor_template.json from +// the reference chart directory. Used when the chart hasn't been saved to the DB yet +// and chartDto.ImageDescriptorTemplate isn't available. Returns an empty string on +// missing file / read error — RenderJson handles an empty template gracefully. +func (impl DeploymentTemplateServiceImpl) readImageDescriptorTemplateFromRefChart(refChartPath string) string { + data, ok := impl.readFileFromRefChart(refChartPath, imageDescriptorTemplateFile) + if !ok { + return "" + } + return string(data) +} + +// readFileFromRefChart scans refChartPath (non-recursive) for the first file whose +// name (case-insensitive) matches one of fileNames and returns its contents. ok=false +// means the path was empty, unreadable, or the file was absent — all logged at the +// appropriate level by the caller's context. +func (impl DeploymentTemplateServiceImpl) readFileFromRefChart(refChartPath string, fileNames ...string) (data []byte, ok bool) { + if len(refChartPath) == 0 { + return nil, false + } + entries, err := os.ReadDir(refChartPath) + if err != nil { + impl.Logger.Errorw("error in reading ref chart dir", "refChartPath", refChartPath, "err", err) + return nil, false + } + targets := make(map[string]struct{}, len(fileNames)) + for _, name := range fileNames { + targets[strings.ToLower(name)] = struct{}{} + } + for _, entry := range entries { + if entry.IsDir() { + continue + } + if _, match := targets[strings.ToLower(entry.Name())]; !match { + continue + } + path := filepath.Clean(filepath.Join(refChartPath, entry.Name())) + contents, err := os.ReadFile(path) + if err != nil { + impl.Logger.Errorw("error in reading file from ref chart", "path", path, "err", err) + return nil, false + } + return contents, true + } + return nil, false +} + +// resolveImageDescriptorTemplate returns the image descriptor template for the app's +// chart. It prefers the saved chart in DB; if no chart has been saved for this +// (appId, chartRefId) pair (pg.ErrNoRows), it falls back to the reference chart's +// .image_descriptor_template.json on disk so manifest rendering works before save. +// Any other DB error is returned to the caller. +func (impl DeploymentTemplateServiceImpl) resolveImageDescriptorTemplate(appId, chartRefId int, refChartPath string) (string, error) { + chartDto, err := impl.chartReadService.GetByAppIdAndChartRefId(appId, chartRefId) + if err == nil { + return chartDto.ImageDescriptorTemplate, nil + } + if !errors.Is(err, pg.ErrNoRows) { + impl.Logger.Errorw("error in getting chart by appId and chartRefId", "appId", appId, "chartRefId", chartRefId, "err", err) + return "", err + } + return impl.readImageDescriptorTemplateFromRefChart(refChartPath), nil +} + // mergeReleaseOverrideIntoValuesYaml merges the chart's ReleaseOverride JSON // on top of the given values YAML. On any failure the input valuesYaml is // returned unchanged so template rendering can still proceed. From 5b3426e3e809c5934e730e155139a2de855e3f71 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 5 Aug 2026 14:01:53 +0530 Subject: [PATCH 03/12] sql script sync --- ...600_add_ci_artifact_query_indexes.down.sql | 8 ++ ...04600_add_ci_artifact_query_indexes.up.sql | 77 +++++++++++++++++++ ...dd_ci_workflow_git_triggers_index.down.sql | 1 + ..._add_ci_workflow_git_triggers_index.up.sql | 23 ++++++ 4 files changed, 109 insertions(+) create mode 100644 scripts/sql/36304600_add_ci_artifact_query_indexes.down.sql create mode 100644 scripts/sql/36304600_add_ci_artifact_query_indexes.up.sql create mode 100644 scripts/sql/36404600_add_ci_workflow_git_triggers_index.down.sql create mode 100644 scripts/sql/36404600_add_ci_workflow_git_triggers_index.up.sql diff --git a/scripts/sql/36304600_add_ci_artifact_query_indexes.down.sql b/scripts/sql/36304600_add_ci_artifact_query_indexes.down.sql new file mode 100644 index 0000000000..8a610b15ab --- /dev/null +++ b/scripts/sql/36304600_add_ci_artifact_query_indexes.down.sql @@ -0,0 +1,8 @@ +-- Rollback: drop indexes added by 36204600_add_ci_artifact_query_indexes.up.sql + +DROP INDEX IF EXISTS idx_ci_artifact_external_ci_pipeline_id; +DROP INDEX IF EXISTS idx_ci_artifact_pipeline_id; +DROP INDEX IF EXISTS idx_ci_artifact_component_id_data_source; +DROP INDEX IF EXISTS idx_ci_workflow_ci_pipeline_id; +DROP INDEX IF EXISTS idx_dar_pipeline_id_active; +DROP INDEX IF EXISTS idx_wes_ci_execution_lookup; diff --git a/scripts/sql/36304600_add_ci_artifact_query_indexes.up.sql b/scripts/sql/36304600_add_ci_artifact_query_indexes.up.sql new file mode 100644 index 0000000000..88df30f843 --- /dev/null +++ b/scripts/sql/36304600_add_ci_artifact_query_indexes.up.sql @@ -0,0 +1,77 @@ +-- Hotfix: add indexes for ci_artifact / ci_workflow / workflow_execution_stage / +-- deployment_approval_request to eliminate sequential scans that were causing +-- 60s PG_READ_TIMEOUT errors on the following hot queries: +-- * CiArtifactRepositoryImpl.GetArtifactsByCDPipelineV3 (external CI + UNION variants) +-- * CiArtifactRepositoryImpl.FindArtifactByListFilter +-- * CiWorkflowRepositoryImpl.FindByPipelineId +-- * CiWorkflowRepositoryImpl.FindByStatusesIn +-- * WorkflowStageRepositoryImpl.GetSuccessfulCIExecutionStages +-- +-- Note: CREATE INDEX CONCURRENTLY cannot run inside a transaction (golang-migrate +-- wraps each migration in a transaction). We use plain CREATE INDEX IF NOT EXISTS +-- here so the migration is idempotent. For large existing tables, operators +-- SHOULD create these indexes manually with CONCURRENTLY BEFORE deploying this +-- release to avoid blocking writes during pod startup: +-- +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ci_artifact_external_ci_pipeline_id +-- ON ci_artifact (external_ci_pipeline_id) WHERE external_ci_pipeline_id IS NOT NULL; +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ci_artifact_pipeline_id +-- ON ci_artifact (pipeline_id) WHERE pipeline_id IS NOT NULL; +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ci_artifact_component_id_data_source +-- ON ci_artifact (component_id, data_source) WHERE component_id IS NOT NULL; +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ci_workflow_ci_pipeline_id +-- ON ci_workflow (ci_pipeline_id); +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_dar_pipeline_id_active +-- ON deployment_approval_request (pipeline_id) WHERE active = true; +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_wes_ci_execution_lookup +-- ON workflow_execution_stage (created_on) +-- WHERE workflow_type = 'CI' AND stage_name = 'Execution' +-- AND status = 'SUCCEEDED' AND status_for = 'workflow'; +-- +-- Once the indexes exist, this migration is a no-op (IF NOT EXISTS). + +-- 1. ci_artifact.external_ci_pipeline_id +-- Filter used by GetArtifactsByCDPipelineV3 (external webhook CI variant). +-- Partial index: most rows are internal CI (external_ci_pipeline_id IS NULL), +-- so a partial index is much smaller and covers the actual query. +CREATE INDEX IF NOT EXISTS idx_ci_artifact_external_ci_pipeline_id + ON ci_artifact (external_ci_pipeline_id) + WHERE external_ci_pipeline_id IS NOT NULL; + +-- 2. ci_artifact.pipeline_id +-- Filter used by GetArtifactsByCDPipelineV3 (UNION variant) and several other +-- repository methods (Where pipeline_id = ?). +CREATE INDEX IF NOT EXISTS idx_ci_artifact_pipeline_id + ON ci_artifact (pipeline_id) + WHERE pipeline_id IS NOT NULL; + +-- 3. ci_artifact.(component_id, data_source) +-- Composite filter used by GetArtifactsByCDPipelineV3 (UNION variant) and +-- FindArtifactByListFilter: (component_id = ? AND data_source = 'pre_cd'|'post_ci'). +CREATE INDEX IF NOT EXISTS idx_ci_artifact_component_id_data_source + ON ci_artifact (component_id, data_source) + WHERE component_id IS NOT NULL; + +-- 4. ci_workflow.ci_pipeline_id +-- Filter used by FindByPipelineId (WHERE ci_pipeline_id = ? ORDER BY started_on DESC). +CREATE INDEX IF NOT EXISTS idx_ci_workflow_ci_pipeline_id + ON ci_workflow (ci_pipeline_id); + +-- 5. deployment_approval_request.pipeline_id (partial on active = true) +-- Filter used by FindArtifactByListFilter NOT IN subquery +-- (WHERE pipeline_id = ? AND active = true AND artifact_deployment_triggered = false). +-- Partial index on active = true keeps it small. +CREATE INDEX IF NOT EXISTS idx_dar_pipeline_id_active + ON deployment_approval_request (pipeline_id) + WHERE active = true; + +-- 6. workflow_execution_stage covering the exact GetSuccessfulCIExecutionStages filter +-- (workflow_type='CI' AND stage_name='Execution' AND status='SUCCEEDED' +-- AND status_for='workflow' AND created_on BETWEEN ? AND ?). +-- Highly selective partial index shaped to that query. +CREATE INDEX IF NOT EXISTS idx_wes_ci_execution_lookup + ON workflow_execution_stage (created_on) + WHERE workflow_type = 'CI' + AND stage_name = 'Execution' + AND status = 'SUCCEEDED' + AND status_for = 'workflow'; diff --git a/scripts/sql/36404600_add_ci_workflow_git_triggers_index.down.sql b/scripts/sql/36404600_add_ci_workflow_git_triggers_index.down.sql new file mode 100644 index 0000000000..3b18e67b0f --- /dev/null +++ b/scripts/sql/36404600_add_ci_workflow_git_triggers_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_ci_workflow_git_triggers_gin; diff --git a/scripts/sql/36404600_add_ci_workflow_git_triggers_index.up.sql b/scripts/sql/36404600_add_ci_workflow_git_triggers_index.up.sql new file mode 100644 index 0000000000..b97e14d842 --- /dev/null +++ b/scripts/sql/36404600_add_ci_workflow_git_triggers_index.up.sql @@ -0,0 +1,23 @@ +-- Supports CiWorkflowRepositoryImpl.FindWorkflowByMaterialAndCommit, used by +-- validateBuildSequence to check a pipeline's full build history (not just the +-- last build) for whether a given material/commit combination has already been +-- built, before rejecting a stale/redelivered automatic CI trigger. +-- +-- git_triggers is stored as `json`, not `jsonb`, so a plain GIN index can't be +-- created directly on the column. We index the jsonb-cast expression instead, +-- using jsonb_path_ops (smaller and faster than the default jsonb_ops for +-- containment-only (@>) queries, which is the only operator this lookup uses). +-- +-- Note: CREATE INDEX CONCURRENTLY cannot run inside a transaction (golang-migrate +-- wraps each migration in a transaction). We use plain CREATE INDEX IF NOT EXISTS +-- here so the migration is idempotent. For large existing ci_workflow tables, +-- operators SHOULD create this index manually with CONCURRENTLY BEFORE deploying +-- this release, to avoid blocking writes during pod startup: +-- +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ci_workflow_git_triggers_gin +-- ON ci_workflow USING gin ((git_triggers::jsonb) jsonb_path_ops); +-- +-- Once the index exists, this migration is a no-op (IF NOT EXISTS). + +CREATE INDEX IF NOT EXISTS idx_ci_workflow_git_triggers_gin + ON ci_workflow USING gin ((git_triggers::jsonb) jsonb_path_ops); From 5e19f16eba34106b5d02355646ae4b9411eabfc7 Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 2 Sep 2026 14:54:28 +0530 Subject: [PATCH 04/12] looking up live usages of a scoped variable by name , so that users dont delete currently being used vars --- .../ScopedVariableRestHandler.go | 39 ++- api/router/ScopedVariableRouter.go | 3 + pkg/variables/ScopedVariableService.go | 99 +++++- .../ScopedVariableUsageGuard_test.go | 318 ++++++++++++++++++ pkg/variables/VariableEntityMappingService.go | 51 +++ .../VariableEntityMappingService_test.go | 4 +- .../mocks/VariableEntityMappingRepository.go | 34 +- .../mocks/VariableEntityMappingService.go | 45 ++- pkg/variables/models/variableUsage.go | 75 +++++ .../parsers/VariableTemplateParser_test.go | 10 +- .../VariableEntityMappingRepository.go | 82 +++++ .../VariableEntityMappingRepository_test.go | 26 +- ...600_variable_entity_mapping_index.down.sql | 1 + ...04600_variable_entity_mapping_index.up.sql | 4 + wire_gen.go | 6 +- 15 files changed, 761 insertions(+), 36 deletions(-) create mode 100644 pkg/variables/ScopedVariableUsageGuard_test.go create mode 100644 pkg/variables/models/variableUsage.go create mode 100644 scripts/sql/36504600_variable_entity_mapping_index.down.sql create mode 100644 scripts/sql/36504600_variable_entity_mapping_index.up.sql diff --git a/api/restHandler/scopedVariable/ScopedVariableRestHandler.go b/api/restHandler/scopedVariable/ScopedVariableRestHandler.go index 5a8ed20c4d..049d38542b 100644 --- a/api/restHandler/scopedVariable/ScopedVariableRestHandler.go +++ b/api/restHandler/scopedVariable/ScopedVariableRestHandler.go @@ -24,6 +24,7 @@ import ( "strconv" "github.com/devtron-labs/devtron/api/restHandler/common" + util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user" "github.com/devtron-labs/devtron/pkg/bean" @@ -42,6 +43,7 @@ type ScopedVariableRestHandler interface { CreateVariables(w http.ResponseWriter, r *http.Request) GetScopedVariables(w http.ResponseWriter, r *http.Request) GetJsonForVariables(w http.ResponseWriter, r *http.Request) + GetVariableUsage(w http.ResponseWriter, r *http.Request) } type ScopedVariableRestHandlerImpl struct { @@ -108,7 +110,17 @@ func (handler *ScopedVariableRestHandlerImpl) CreateVariables(w http.ResponseWri //RBAC enforcer Ends err = handler.scopedVariableService.CreateVariables(payload) if err != nil { - if errors.As(err, &models.ValidationError{}) { + var deletionBlockedErr *models.VariableDeletionBlockedError + if errors.As(err, &deletionBlockedErr) { + apiErr := &util2.ApiError{ + HttpStatusCode: http.StatusConflict, + Code: strconv.Itoa(http.StatusConflict), + InternalMessage: deletionBlockedErr.Error(), + UserMessage: deletionBlockedErr.Error(), + UserDetailMessage: deletionBlockedErr.UsageSummary(20), + } + common.WriteJsonResp(w, apiErr, nil, http.StatusConflict) + } else if errors.As(err, &models.ValidationError{}) { common.WriteJsonResp(w, err, nil, http.StatusNotAcceptable) } else { common.WriteJsonResp(w, err, nil, http.StatusBadRequest) @@ -117,6 +129,31 @@ func (handler *ScopedVariableRestHandlerImpl) CreateVariables(w http.ResponseWri } common.WriteJsonResp(w, nil, nil, http.StatusOK) } + +func (handler *ScopedVariableRestHandlerImpl) GetVariableUsage(w http.ResponseWriter, r *http.Request) { + userId, err := handler.userAuthService.GetLoggedInUser(r) + if userId == 0 || err != nil { + common.HandleUnauthorized(w, r) + return + } + + // RBAC enforcer applying + token := r.Header.Get("token") + if isSuperAdmin := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !isSuperAdmin { + common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) + return + } + //RBAC enforcer Ends + + variableName := r.URL.Query().Get("name") + usages, err := handler.scopedVariableService.GetVariableUsage(variableName) + if err != nil { + handler.logger.Errorw("service err, GetVariableUsage", "variableName", variableName, "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + common.WriteJsonResp(w, nil, usages, http.StatusOK) +} func (handler *ScopedVariableRestHandlerImpl) GetScopedVariables(w http.ResponseWriter, r *http.Request) { token := r.Header.Get("token") isSuperAdmin := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*") diff --git a/api/router/ScopedVariableRouter.go b/api/router/ScopedVariableRouter.go index 4054f1faef..ba60c99a25 100644 --- a/api/router/ScopedVariableRouter.go +++ b/api/router/ScopedVariableRouter.go @@ -46,5 +46,8 @@ func (impl ScopedVariableRouterImpl) InitScopedVariableRouter(router *mux.Router router.Path("/variables/detail"). HandlerFunc(impl.scopedVariableRestHandler.GetJsonForVariables). Methods("GET") + router.Path("/variables/usage"). + HandlerFunc(impl.scopedVariableRestHandler.GetVariableUsage). + Methods("GET") } diff --git a/pkg/variables/ScopedVariableService.go b/pkg/variables/ScopedVariableService.go index bfde4840b9..5385dee5b5 100644 --- a/pkg/variables/ScopedVariableService.go +++ b/pkg/variables/ScopedVariableService.go @@ -49,25 +49,28 @@ type ScopedVariableService interface { GetFormattedVariableForName(name string) string GetMatchedScopedVariables(varScope []*resourceQualifiers.QualifierMapping) map[int][]*resourceQualifiers.QualifierMapping GetScopeWithPriority(variableIdToVariableScopes map[int][]*resourceQualifiers.QualifierMapping) map[int]int + GetVariableUsage(variableName string) ([]*models.VariableUsage, error) } type ScopedVariableServiceImpl struct { - logger *zap.SugaredLogger - scopedVariableRepository repository2.ScopedVariableRepository - qualifierMappingService resourceQualifiers.QualifierMappingService - VariableNameConfig *VariableConfig - VariableCache *cache.VariableCacheObj - asyncRunnable *async.Runnable + logger *zap.SugaredLogger + scopedVariableRepository repository2.ScopedVariableRepository + qualifierMappingService resourceQualifiers.QualifierMappingService + variableEntityMappingService VariableEntityMappingService + VariableNameConfig *VariableConfig + VariableCache *cache.VariableCacheObj + asyncRunnable *async.Runnable } func NewScopedVariableServiceImpl(logger *zap.SugaredLogger, scopedVariableRepository repository2.ScopedVariableRepository, appRepository app.AppRepository, environmentRepository repository3.EnvironmentRepository, devtronResourceSearchableKeyService read.DevtronResourceSearchableKeyService, clusterRepository repository.ClusterRepository, - qualifierMappingService resourceQualifiers.QualifierMappingService, asyncRunnable *async.Runnable) (*ScopedVariableServiceImpl, error) { + qualifierMappingService resourceQualifiers.QualifierMappingService, variableEntityMappingService VariableEntityMappingService, asyncRunnable *async.Runnable) (*ScopedVariableServiceImpl, error) { scopedVariableService := &ScopedVariableServiceImpl{ - logger: logger, - scopedVariableRepository: scopedVariableRepository, - qualifierMappingService: qualifierMappingService, - VariableCache: &cache.VariableCacheObj{CacheLock: &sync.Mutex{}}, - asyncRunnable: asyncRunnable, + logger: logger, + scopedVariableRepository: scopedVariableRepository, + qualifierMappingService: qualifierMappingService, + variableEntityMappingService: variableEntityMappingService, + VariableCache: &cache.VariableCacheObj{CacheLock: &sync.Mutex{}}, + asyncRunnable: asyncRunnable, } cfg, err := GetVariableNameConfig() if err != nil { @@ -156,6 +159,10 @@ func (impl *ScopedVariableServiceImpl) CreateVariables(payload models.Payload) e impl.logger.Errorw("error in variable payload validation", "err", err) return err } + err = impl.checkVariablesToBeDeletedForUsage(payload) + if err != nil { + return err + } auditLog := resourceQualifiers.GetAuditLog(payload.UserId) // Begin Transaction @@ -208,6 +215,74 @@ func (impl *ScopedVariableServiceImpl) CreateVariables(payload models.Payload) e return nil } +// checkVariablesToBeDeletedForUsage blocks the save when it would delete (i.e. the payload no +// longer contains) a variable that is still referenced by live configuration. The whole request +// is rejected so that nothing is deleted partially. +func (impl *ScopedVariableServiceImpl) checkVariablesToBeDeletedForUsage(payload models.Payload) error { + existingVariables, err := impl.scopedVariableRepository.GetAllVariableMetadata() + if err != nil { + impl.logger.Errorw("error in fetching existing variables for deletion check", "err", err) + return err + } + if len(existingVariables) == 0 { + return nil + } + payloadNames := make(map[string]bool, len(payload.Variables)) + for _, variable := range payload.Variables { + payloadNames[variable.Definition.VarName] = true + } + variablesToBeDeleted := make([]string, 0) + for _, existing := range existingVariables { + if !payloadNames[existing.Name] { + variablesToBeDeleted = append(variablesToBeDeleted, existing.Name) + } + } + if len(variablesToBeDeleted) == 0 { + return nil + } + usages, err := impl.variableEntityMappingService.GetLiveVariableUsage(variablesToBeDeleted) + if err != nil { + impl.logger.Errorw("error in fetching variable usages for deletion check", "variables", variablesToBeDeleted, "err", err) + return err + } + if len(usages) == 0 { + return nil + } + blockedVariables := make([]string, 0) + seen := make(map[string]bool) + for _, usage := range usages { + if !seen[usage.VariableName] { + seen[usage.VariableName] = true + blockedVariables = append(blockedVariables, usage.VariableName) + } + } + slices.Sort(blockedVariables) + impl.logger.Warnw("variable deletion blocked, variables are in use", "variables", blockedVariables) + return &models.VariableDeletionBlockedError{ + BlockedVariables: blockedVariables, + Usages: usages, + } +} + +// GetVariableUsage returns the live references of the given variable; when variableName is empty, +// usages of all active variables are returned. +func (impl *ScopedVariableServiceImpl) GetVariableUsage(variableName string) ([]*models.VariableUsage, error) { + variableNames := make([]string, 0) + if variableName != "" { + variableNames = append(variableNames, variableName) + } else { + existingVariables, err := impl.scopedVariableRepository.GetAllVariableMetadata() + if err != nil { + impl.logger.Errorw("error in fetching variables for usage lookup", "err", err) + return nil, err + } + for _, existing := range existingVariables { + variableNames = append(variableNames, existing.Name) + } + } + return impl.variableEntityMappingService.GetLiveVariableUsage(variableNames) +} + func (impl *ScopedVariableServiceImpl) storeVariableData(scopeIdToVarData map[int]string, auditLog sql.AuditLog, tx *pg.Tx) error { VariableDataList := make([]*repository2.VariableData, 0) for scopeId, data := range scopeIdToVarData { diff --git a/pkg/variables/ScopedVariableUsageGuard_test.go b/pkg/variables/ScopedVariableUsageGuard_test.go new file mode 100644 index 0000000000..e7e3757010 --- /dev/null +++ b/pkg/variables/ScopedVariableUsageGuard_test.go @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package variables + +import ( + "errors" + "testing" + + "github.com/devtron-labs/devtron/pkg/sql" + mocks2 "github.com/devtron-labs/devtron/pkg/variables/mocks" + "github.com/devtron-labs/devtron/pkg/variables/models" + repository2 "github.com/devtron-labs/devtron/pkg/variables/repository" + "github.com/go-pg/pg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.uber.org/zap" +) + +// scopedVariableRepoStub satisfies ScopedVariableRepository via interface embedding; only the +// methods used by the deletion guard are implemented, so any other call panics the test. +type scopedVariableRepoStub struct { + repository2.ScopedVariableRepository + metadata []*repository2.VariableDefinition + metadataErr error + deleteCalled bool + txStartCalled bool +} + +func (s *scopedVariableRepoStub) GetAllVariableMetadata() ([]*repository2.VariableDefinition, error) { + return s.metadata, s.metadataErr +} + +func (s *scopedVariableRepoStub) DeleteVariables(auditLog sql.AuditLog, tx *pg.Tx) error { + s.deleteCalled = true + return nil +} + +func (s *scopedVariableRepoStub) StartTx() (*pg.Tx, error) { + s.txStartCalled = true + return nil, errors.New("transaction must not be started when deletion is blocked") +} + +func defsFor(names ...string) []*repository2.VariableDefinition { + defs := make([]*repository2.VariableDefinition, 0, len(names)) + for _, name := range names { + defs = append(defs, &repository2.VariableDefinition{Name: name, Active: true}) + } + return defs +} + +func payloadFor(names ...string) models.Payload { + variables := make([]*models.Variables, 0, len(names)) + for _, name := range names { + variables = append(variables, &models.Variables{ + Definition: models.Definition{VarName: name, DataType: models.PRIMITIVE_TYPE, VarType: models.PUBLIC}, + }) + } + return models.Payload{Variables: variables, UserId: 1} +} + +func usageFor(name string) *models.VariableUsage { + return &models.VariableUsage{ + VariableName: name, + UsageType: models.UsageTypeDeploymentTemplate, + AppId: 1, + AppName: "test-app", + } +} + +func guardTestService(t *testing.T, repoStub *scopedVariableRepoStub) (*ScopedVariableServiceImpl, *mocks2.VariableEntityMappingService) { + logger, err := zap.NewDevelopment() + assert.Nil(t, err) + cfg, err := GetVariableNameConfig() + assert.Nil(t, err) + mappingService := mocks2.NewVariableEntityMappingService(t) + impl := &ScopedVariableServiceImpl{ + logger: logger.Sugar(), + scopedVariableRepository: repoStub, + variableEntityMappingService: mappingService, + VariableNameConfig: cfg, + } + return impl, mappingService +} + +func TestDeletionGuard_FirstSaveNoExistingVariables(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: nil} + impl, mappingService := guardTestService(t, repoStub) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("V1")) + + assert.Nil(t, err) + mappingService.AssertNotCalled(t, "GetLiveVariableUsage", mock.Anything) +} + +func TestDeletionGuard_NoVariableRemoved(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1", "V2")} + impl, mappingService := guardTestService(t, repoStub) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("V1", "V2", "V3")) + + assert.Nil(t, err) + mappingService.AssertNotCalled(t, "GetLiveVariableUsage", mock.Anything) +} + +func TestDeletionGuard_RemovedVariableUnused(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1", "V2")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", []string{"V2"}).Return([]*models.VariableUsage{}, nil) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("V1")) + + assert.Nil(t, err) +} + +func TestDeletionGuard_RemovedVariableInUse(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1", "V2")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", []string{"V2"}). + Return([]*models.VariableUsage{usageFor("V2")}, nil) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("V1")) + + var blockedErr *models.VariableDeletionBlockedError + assert.True(t, errors.As(err, &blockedErr)) + assert.Equal(t, []string{"V2"}, blockedErr.BlockedVariables) + assert.Len(t, blockedErr.Usages, 1) + assert.Contains(t, blockedErr.Error(), "V2") + assert.Contains(t, blockedErr.Error(), "currently in use") +} + +func TestDeletionGuard_MixedRemoval_OnlyUsedOnesBlock(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1", "V2", "V3")} + impl, mappingService := guardTestService(t, repoStub) + // V2 and V3 are removed; only V3 is in use + mappingService.On("GetLiveVariableUsage", mock.MatchedBy(func(names []string) bool { + return len(names) == 2 + })).Return([]*models.VariableUsage{usageFor("V3"), usageFor("V3")}, nil) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("V1")) + + var blockedErr *models.VariableDeletionBlockedError + assert.True(t, errors.As(err, &blockedErr)) + assert.Equal(t, []string{"V3"}, blockedErr.BlockedVariables) +} + +func TestDeletionGuard_EmptyPayloadChecksAllVariables(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1", "V2")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", mock.MatchedBy(func(names []string) bool { + return len(names) == 2 + })).Return([]*models.VariableUsage{usageFor("V2"), usageFor("V1")}, nil) + + err := impl.checkVariablesToBeDeletedForUsage(models.Payload{UserId: 1}) + + var blockedErr *models.VariableDeletionBlockedError + assert.True(t, errors.As(err, &blockedErr)) + // blocked variables are reported sorted + assert.Equal(t, []string{"V1", "V2"}, blockedErr.BlockedVariables) +} + +func TestDeletionGuard_RenameIsTreatedAsDeletion(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("DB_HOST")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", []string{"DB_HOST"}). + Return([]*models.VariableUsage{usageFor("DB_HOST")}, nil) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("DB_HOST_NEW")) + + var blockedErr *models.VariableDeletionBlockedError + assert.True(t, errors.As(err, &blockedErr)) + assert.Equal(t, []string{"DB_HOST"}, blockedErr.BlockedVariables) +} + +func TestDeletionGuard_NamesAreCaseSensitive(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("DB_HOST")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", []string{"DB_HOST"}). + Return([]*models.VariableUsage{usageFor("DB_HOST")}, nil) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("db_host")) + + var blockedErr *models.VariableDeletionBlockedError + assert.True(t, errors.As(err, &blockedErr)) +} + +func TestDeletionGuard_MetadataFetchErrorPropagates(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadataErr: errors.New("db down")} + impl, _ := guardTestService(t, repoStub) + + err := impl.checkVariablesToBeDeletedForUsage(payloadFor("V1")) + + assert.NotNil(t, err) + assert.False(t, repoStub.deleteCalled) +} + +func TestDeletionGuard_UsageLookupErrorPropagates(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", []string{"V1"}).Return(nil, errors.New("db down")) + + err := impl.checkVariablesToBeDeletedForUsage(models.Payload{UserId: 1}) + + assert.NotNil(t, err) + var blockedErr *models.VariableDeletionBlockedError + assert.False(t, errors.As(err, &blockedErr)) +} + +// CreateVariables must reject the whole request before starting the delete-and-recreate +// transaction when a removed variable is still in use. +func TestCreateVariables_BlockedBeforeAnythingIsDeleted(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1", "V2")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", []string{"V1"}). + Return([]*models.VariableUsage{usageFor("V1")}, nil) + + err := impl.CreateVariables(payloadFor("V2")) + + var blockedErr *models.VariableDeletionBlockedError + assert.True(t, errors.As(err, &blockedErr)) + assert.False(t, repoStub.txStartCalled, "transaction must not start when deletion is blocked") + assert.False(t, repoStub.deleteCalled, "no variable may be deleted when deletion is blocked") +} + +func TestCreateVariables_PayloadValidationRunsBeforeUsageCheck(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1")} + impl, mappingService := guardTestService(t, repoStub) + + // duplicate variable names fail payload validation before the usage check + err := impl.CreateVariables(payloadFor("V2", "V2")) + + assert.NotNil(t, err) + assert.True(t, errors.As(err, &models.ValidationError{})) + mappingService.AssertNotCalled(t, "GetLiveVariableUsage", mock.Anything) +} + +func TestGetVariableUsage_ForSingleVariable(t *testing.T) { + repoStub := &scopedVariableRepoStub{} + impl, mappingService := guardTestService(t, repoStub) + expected := []*models.VariableUsage{usageFor("V1")} + mappingService.On("GetLiveVariableUsage", []string{"V1"}).Return(expected, nil) + + usages, err := impl.GetVariableUsage("V1") + + assert.Nil(t, err) + assert.Equal(t, expected, usages) +} + +func TestGetVariableUsage_AllVariablesWhenNameEmpty(t *testing.T) { + repoStub := &scopedVariableRepoStub{metadata: defsFor("V1", "V2")} + impl, mappingService := guardTestService(t, repoStub) + mappingService.On("GetLiveVariableUsage", []string{"V1", "V2"}). + Return([]*models.VariableUsage{usageFor("V1")}, nil) + + usages, err := impl.GetVariableUsage("") + + assert.Nil(t, err) + assert.Len(t, usages, 1) +} + +func TestGetLiveVariableUsage_MapsEntityTypesAndDedupes(t *testing.T) { + logger, err := zap.NewDevelopment() + assert.Nil(t, err) + repoMock := mocks2.NewVariableEntityMappingRepository(t) + service := NewVariableEntityMappingServiceImpl(repoMock, logger.Sugar()) + + rows := []*repository2.VariableUsageRow{ + {VariableName: "V1", EntityType: repository2.EntityTypeDeploymentTemplateAppLevel, EntityId: 10, AppId: 1, AppName: "app-1"}, + // duplicate mapping row for the same entity must be deduped + {VariableName: "V1", EntityType: repository2.EntityTypeDeploymentTemplateAppLevel, EntityId: 10, AppId: 1, AppName: "app-1"}, + {VariableName: "V1", EntityType: repository2.EntityTypeSecretEnvLevel, EntityId: 20, AppId: 1, AppName: "app-1", EnvId: 3, EnvName: "env-3"}, + {VariableName: "V2", EntityType: repository2.EntityTypeConfigMapAppLevel, EntityId: 30, AppId: 2, AppName: "app-2"}, + {VariableName: "V2", EntityType: repository2.EntityTypePipelineStage, EntityId: 40, AppId: 2, AppName: "app-2", PipelineName: "ci-pipeline", StageType: "PRE_CI"}, + // two different variables on the same entity must both be reported + {VariableName: "V3", EntityType: repository2.EntityTypeDeploymentTemplateAppLevel, EntityId: 10, AppId: 1, AppName: "app-1"}, + } + repoMock.On("GetLiveUsagesForVariableNames", []string{"V1", "V2", "V3"}).Return(rows, nil) + + usages, err := service.GetLiveVariableUsage([]string{"V1", "V2", "V3"}) + + assert.Nil(t, err) + assert.Len(t, usages, 5) + usageTypeByKey := make(map[string]models.VariableUsageType) + for _, usage := range usages { + usageTypeByKey[usage.VariableName+"/"+string(usage.UsageType)] = usage.UsageType + } + assert.Equal(t, models.UsageTypeDeploymentTemplate, usageTypeByKey["V1/deploymentTemplate"]) + assert.Equal(t, models.UsageTypeSecret, usageTypeByKey["V1/secret"]) + assert.Equal(t, models.UsageTypeConfigMap, usageTypeByKey["V2/configMap"]) + assert.Equal(t, models.UsageTypePipelineStage, usageTypeByKey["V2/pipelineStage"]) + assert.Equal(t, models.UsageTypeDeploymentTemplate, usageTypeByKey["V3/deploymentTemplate"]) +} + +func TestGetLiveVariableUsage_EmptyInputSkipsRepository(t *testing.T) { + logger, err := zap.NewDevelopment() + assert.Nil(t, err) + repoMock := mocks2.NewVariableEntityMappingRepository(t) + service := NewVariableEntityMappingServiceImpl(repoMock, logger.Sugar()) + + usages, err := service.GetLiveVariableUsage(nil) + + assert.Nil(t, err) + assert.Empty(t, usages) + repoMock.AssertNotCalled(t, "GetLiveUsagesForVariableNames", mock.Anything) +} diff --git a/pkg/variables/VariableEntityMappingService.go b/pkg/variables/VariableEntityMappingService.go index 8835e2a514..e6b1517f69 100644 --- a/pkg/variables/VariableEntityMappingService.go +++ b/pkg/variables/VariableEntityMappingService.go @@ -19,6 +19,7 @@ package variables import ( mapset "github.com/deckarep/golang-set" "github.com/devtron-labs/devtron/pkg/sql" + "github.com/devtron-labs/devtron/pkg/variables/models" "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/pkg/variables/utils" "github.com/go-pg/pg" @@ -29,6 +30,7 @@ import ( type VariableEntityMappingService interface { UpdateVariablesForEntity(variableNames []string, entity repository.Entity, userId int32, tx *pg.Tx) error GetAllMappingsForEntities(entities []repository.Entity) (map[repository.Entity][]string, error) + GetLiveVariableUsage(variableNames []string) ([]*models.VariableUsage, error) DeleteMappingsForEntities(entities []repository.Entity, userId int32, tx *pg.Tx) error } @@ -135,6 +137,55 @@ func (impl VariableEntityMappingServiceImpl) GetAllMappingsForEntities(entities return entityIdToVariableNames, nil } +func getUsageTypeForEntityType(entityType repository.EntityType) models.VariableUsageType { + switch entityType { + case repository.EntityTypeDeploymentTemplateAppLevel, repository.EntityTypeDeploymentTemplateEnvLevel: + return models.UsageTypeDeploymentTemplate + case repository.EntityTypeConfigMapAppLevel, repository.EntityTypeConfigMapEnvLevel: + return models.UsageTypeConfigMap + case repository.EntityTypeSecretAppLevel, repository.EntityTypeSecretEnvLevel: + return models.UsageTypeSecret + case repository.EntityTypePipelineStage: + return models.UsageTypePipelineStage + } + return "" +} + +func (impl VariableEntityMappingServiceImpl) GetLiveVariableUsage(variableNames []string) ([]*models.VariableUsage, error) { + usages := make([]*models.VariableUsage, 0) + if len(variableNames) == 0 { + return usages, nil + } + usageRows, err := impl.variableEntityMappingRepository.GetLiveUsagesForVariableNames(variableNames) + if err != nil { + impl.logger.Errorw("error in fetching live usages for variables", "variableNames", variableNames, "err", err) + return nil, err + } + // the mapping table has no unique constraint, dedupe on (name, entityType, entityId) + seen := make(map[repository.Entity]map[string]bool) + for _, row := range usageRows { + entity := repository.GetEntity(row.EntityId, row.EntityType) + if seen[entity] == nil { + seen[entity] = make(map[string]bool) + } + if seen[entity][row.VariableName] { + continue + } + seen[entity][row.VariableName] = true + usages = append(usages, &models.VariableUsage{ + VariableName: row.VariableName, + UsageType: getUsageTypeForEntityType(row.EntityType), + AppId: row.AppId, + AppName: row.AppName, + EnvId: row.EnvId, + EnvName: row.EnvName, + PipelineName: row.PipelineName, + StageType: row.StageType, + }) + } + return usages, nil +} + func (impl VariableEntityMappingServiceImpl) DeleteMappingsForEntities(entities []repository.Entity, userId int32, tx *pg.Tx) error { err := impl.variableEntityMappingRepository.DeleteAllVariablesForEntities(tx, entities, userId) if err != nil { diff --git a/pkg/variables/VariableEntityMappingService_test.go b/pkg/variables/VariableEntityMappingService_test.go index 176c45589d..17e36a75e4 100644 --- a/pkg/variables/VariableEntityMappingService_test.go +++ b/pkg/variables/VariableEntityMappingService_test.go @@ -67,7 +67,7 @@ func TestVariableEntityMappingServiceImpl_DeleteMappingsForEntities(t *testing.T logger: tt.fields.logger, variableEntityMappingRepository: tt.fields.variableEntityMappingRepository, } - if err := impl.DeleteMappingsForEntities(tt.args.entities, tt.args.userId); (err != nil) != tt.wantErr { + if err := impl.DeleteMappingsForEntities(tt.args.entities, tt.args.userId, nil); (err != nil) != tt.wantErr { t.Errorf("DeleteMappingsForEntities() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -133,7 +133,7 @@ func TestVariableEntityMappingServiceImpl_UpdateVariablesForEntity(t *testing.T) logger: tt.fields.logger, variableEntityMappingRepository: tt.fields.variableEntityMappingRepository, } - if err := impl.UpdateVariablesForEntity(tt.args.variableNames, tt.args.entity, tt.args.userId); (err != nil) != tt.wantErr { + if err := impl.UpdateVariablesForEntity(tt.args.variableNames, tt.args.entity, tt.args.userId, nil); (err != nil) != tt.wantErr { t.Errorf("UpdateVariablesForEntity() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pkg/variables/mocks/VariableEntityMappingRepository.go b/pkg/variables/mocks/VariableEntityMappingRepository.go index 990b23e396..265a843f31 100644 --- a/pkg/variables/mocks/VariableEntityMappingRepository.go +++ b/pkg/variables/mocks/VariableEntityMappingRepository.go @@ -29,12 +29,12 @@ func (_m *VariableEntityMappingRepository) CommitTx(tx *pg.Tx) error { } // DeleteAllVariablesForEntities provides a mock function with given fields: entities, userId -func (_m *VariableEntityMappingRepository) DeleteAllVariablesForEntities(entities []repository.Entity, userId int32) error { - ret := _m.Called(entities, userId) +func (_m *VariableEntityMappingRepository) DeleteAllVariablesForEntities(tx *pg.Tx, entities []repository.Entity, userId int32) error { + ret := _m.Called(tx, entities, userId) var r0 error - if rf, ok := ret.Get(0).(func([]repository.Entity, int32) error); ok { - r0 = rf(entities, userId) + if rf, ok := ret.Get(0).(func(*pg.Tx, []repository.Entity, int32) error); ok { + r0 = rf(tx, entities, userId) } else { r0 = ret.Error(0) } @@ -56,6 +56,32 @@ func (_m *VariableEntityMappingRepository) DeleteVariablesForEntity(tx *pg.Tx, v return r0 } +// GetLiveUsagesForVariableNames provides a mock function with given fields: variableNames +func (_m *VariableEntityMappingRepository) GetLiveUsagesForVariableNames(variableNames []string) ([]*repository.VariableUsageRow, error) { + ret := _m.Called(variableNames) + + var r0 []*repository.VariableUsageRow + var r1 error + if rf, ok := ret.Get(0).(func([]string) ([]*repository.VariableUsageRow, error)); ok { + return rf(variableNames) + } + if rf, ok := ret.Get(0).(func([]string) []*repository.VariableUsageRow); ok { + r0 = rf(variableNames) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*repository.VariableUsageRow) + } + } + + if rf, ok := ret.Get(1).(func([]string) error); ok { + r1 = rf(variableNames) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // GetVariablesForEntities provides a mock function with given fields: entities func (_m *VariableEntityMappingRepository) GetVariablesForEntities(entities []repository.Entity) ([]*repository.VariableEntityMapping, error) { ret := _m.Called(entities) diff --git a/pkg/variables/mocks/VariableEntityMappingService.go b/pkg/variables/mocks/VariableEntityMappingService.go index 345edf6ea5..759d1dc2f0 100644 --- a/pkg/variables/mocks/VariableEntityMappingService.go +++ b/pkg/variables/mocks/VariableEntityMappingService.go @@ -3,6 +3,9 @@ package mocks import ( + pg "github.com/go-pg/pg" + + models "github.com/devtron-labs/devtron/pkg/variables/models" repository "github.com/devtron-labs/devtron/pkg/variables/repository" mock "github.com/stretchr/testify/mock" ) @@ -13,12 +16,12 @@ type VariableEntityMappingService struct { } // DeleteMappingsForEntities provides a mock function with given fields: entities, userId -func (_m *VariableEntityMappingService) DeleteMappingsForEntities(entities []repository.Entity, userId int32) error { - ret := _m.Called(entities, userId) +func (_m *VariableEntityMappingService) DeleteMappingsForEntities(entities []repository.Entity, userId int32, tx *pg.Tx) error { + ret := _m.Called(entities, userId, tx) var r0 error - if rf, ok := ret.Get(0).(func([]repository.Entity, int32) error); ok { - r0 = rf(entities, userId) + if rf, ok := ret.Get(0).(func([]repository.Entity, int32, *pg.Tx) error); ok { + r0 = rf(entities, userId, tx) } else { r0 = ret.Error(0) } @@ -52,13 +55,39 @@ func (_m *VariableEntityMappingService) GetAllMappingsForEntities(entities []rep return r0, r1 } +// GetLiveVariableUsage provides a mock function with given fields: variableNames +func (_m *VariableEntityMappingService) GetLiveVariableUsage(variableNames []string) ([]*models.VariableUsage, error) { + ret := _m.Called(variableNames) + + var r0 []*models.VariableUsage + var r1 error + if rf, ok := ret.Get(0).(func([]string) ([]*models.VariableUsage, error)); ok { + return rf(variableNames) + } + if rf, ok := ret.Get(0).(func([]string) []*models.VariableUsage); ok { + r0 = rf(variableNames) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*models.VariableUsage) + } + } + + if rf, ok := ret.Get(1).(func([]string) error); ok { + r1 = rf(variableNames) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // UpdateVariablesForEntity provides a mock function with given fields: variableNames, entity, userId -func (_m *VariableEntityMappingService) UpdateVariablesForEntity(variableNames []string, entity repository.Entity, userId int32) error { - ret := _m.Called(variableNames, entity, userId) +func (_m *VariableEntityMappingService) UpdateVariablesForEntity(variableNames []string, entity repository.Entity, userId int32, tx *pg.Tx) error { + ret := _m.Called(variableNames, entity, userId, tx) var r0 error - if rf, ok := ret.Get(0).(func([]string, repository.Entity, int32) error); ok { - r0 = rf(variableNames, entity, userId) + if rf, ok := ret.Get(0).(func([]string, repository.Entity, int32, *pg.Tx) error); ok { + r0 = rf(variableNames, entity, userId, tx) } else { r0 = ret.Error(0) } diff --git a/pkg/variables/models/variableUsage.go b/pkg/variables/models/variableUsage.go new file mode 100644 index 0000000000..b7d1ae72da --- /dev/null +++ b/pkg/variables/models/variableUsage.go @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package models + +import ( + "fmt" + "strings" +) + +type VariableUsageType string + +const ( + UsageTypeDeploymentTemplate VariableUsageType = "deploymentTemplate" + UsageTypeConfigMap VariableUsageType = "configMap" + UsageTypeSecret VariableUsageType = "secret" + UsageTypePipelineStage VariableUsageType = "pipelineStage" + UsageTypeInfraConfig VariableUsageType = "infraConfig" +) + +// VariableUsage is one live reference of a scoped variable in saved configuration. +type VariableUsage struct { + VariableName string `json:"variableName"` + UsageType VariableUsageType `json:"usageType"` + AppId int `json:"appId,omitempty"` + AppName string `json:"appName,omitempty"` + EnvId int `json:"envId,omitempty"` + EnvName string `json:"envName,omitempty"` + PipelineName string `json:"pipelineName,omitempty"` + StageType string `json:"stageType,omitempty"` +} + +// VariableDeletionBlockedError is returned when a variable save would delete variables +// that are still referenced by live configuration. +type VariableDeletionBlockedError struct { + BlockedVariables []string `json:"blockedVariables"` + Usages []*VariableUsage `json:"usages"` +} + +func (err *VariableDeletionBlockedError) Error() string { + return fmt.Sprintf("variable(s) [%s] cannot be deleted because they are currently in use; remove their references first", strings.Join(err.BlockedVariables, ", ")) +} + +// UsageSummary renders at most limit usages as a human-readable list for error messages. +func (err *VariableDeletionBlockedError) UsageSummary(limit int) string { + parts := make([]string, 0, len(err.Usages)) + for _, usage := range err.Usages { + if len(parts) == limit { + parts = append(parts, fmt.Sprintf("and %d more", len(err.Usages)-limit)) + break + } + location := usage.AppName + if usage.EnvName != "" { + location = location + "/" + usage.EnvName + } + if usage.PipelineName != "" { + location = location + "/" + usage.PipelineName + } + parts = append(parts, fmt.Sprintf("%s is used in %s (%s)", usage.VariableName, location, usage.UsageType)) + } + return strings.Join(parts, "; ") +} diff --git a/pkg/variables/parsers/VariableTemplateParser_test.go b/pkg/variables/parsers/VariableTemplateParser_test.go index 0a6241a4aa..f35c333754 100644 --- a/pkg/variables/parsers/VariableTemplateParser_test.go +++ b/pkg/variables/parsers/VariableTemplateParser_test.go @@ -31,13 +31,15 @@ const StringTemplateWithIntParam = "- EXTERNAL_CI_ID: \"1\"\n REPO_NAME_EXTERNA const StringTemplateWithIntParamResolvedTemplate = "- EXTERNAL_CI_ID: \"1\"\n REPO_NAME_EXTERNAL_CI: 1800\n REGISTRY_URL_EXTERNAL_CI: docker.io/shivamnagar409\n- EXTERNAL_CI_ID: \"2\"\n REPO_NAME_EXTERNAL_CI: test123\n REGISTRY_URL_EXTERNAL_CI: docker.io/shivamnagar409\n" func TestVariableTemplateParserImpl_ExtractVariables(t *testing.T) { + t.Setenv("SCOPED_VARIABLE_ENABLED", "true") logger, err := util.NewSugardLogger() assert.Nil(t, err) t.Run("extract variables", func(t *testing.T) { - templateParser := NewVariableTemplateParserImpl(logger) // \"value\" + templateParser, err := NewVariableTemplateParserImpl(logger) + assert.Nil(t, err) sampleTemplate := `{"ConfigMaps":{"enabled":false,"maps":[]},"ConfigSecrets":{"enabled":false,"secrets":[]},"ContainerPort":[{"envoyPort":"@{{envoyPort + 0}}","idleTimeout":"@{{idleTimeoutVar / idleTimeoutDivVar}}s","name":"${1 + appName}","port":8080,"servicePort":80,"supportStreaming":false,"useHTTP2":false}],"EnvVariables":[],"EnvVariablesFromFieldPath":[{"fieldPath":"metadata.name","name":"POD_NAME"}],"GracePeriod":30,"LivenessProbe":{"Path":"","command":[],"failureThreshold":3,"httpHeaders":[],"initialDelaySeconds":20,"periodSeconds":10,"port":8080,"scheme":"","successThreshold":1,"tcp":false,"timeoutSeconds":5},"MaxSurge":1,"MaxUnavailable":0,"MinReadySeconds":60,"ReadinessProbe":{"Path":"","command":[],"failureThreshold":3,"httpHeaders":[],"initialDelaySeconds":20,"periodSeconds":10,"port":8080,"scheme":"","successThreshold":1,"tcp":false,"timeoutSeconds":5},"Spec":{"Affinity":{"Values":"nodes","key":""}},"ambassadorMapping":{"ambassadorId":"","cors":{},"enabled":false,"hostname":"devtron.example.com","labels":{},"prefix":"/","retryPolicy":{},"rewrite":"","tls":{"context":"","create":false,"hosts":[],"secretName":""}},"args":{"enabled":false,"value":["/bin/sh","-c","touch /tmp/healthy; sleep 30; rm -rf /tmp/healthy; sleep 600"]},"autoPromotionSeconds":30,"autoscaling":{"MaxReplicas":2,"MinReplicas":1,"TargetCPUUtilizationPercentage":90,"TargetMemoryUtilizationPercentage":80,"annotations":{},"behavior":{},"enabled":false,"extraMetrics":[],"labels":{}},"command":{"enabled":false,"value":[],"workingDir":{}},"containerExtraSpecs":{},"containerSecurityContext":{},"containerSpec":{"lifecycle":{"enabled":false,"postStart":{"httpGet":{"host":"example.com","path":"/example","port":90}},"preStop":{"exec":{"command":["sleep","10"]}}}},"containers":[],"dbMigrationConfig":{"enabled":false},"envoyproxy":{"configMapName":"","image":"quay.io/devtron/envoy:v1.14.1","lifecycle":{},"resources":{"limits":{"cpu":"50m","memory":"50Mi"},"requests":{"cpu":"50m","memory":"50Mi"}}},"hostAliases":[],"image":{"pullPolicy":"IfNotPresent"},"imagePullSecrets":[],"ingress":{"annotations":{},"className":"","enabled":false,"hosts":[{"host":"chart-example1.local","pathType":"ImplementationSpecific","paths":["/example1"]},{"host":"chart-example2.local","pathType":"ImplementationSpecific","paths":["/example2","/example2/healthz"]}],"labels":{},"tls":[]},"ingressInternal":{"annotations":{},"className":"","enabled":false,"hosts":[{"host":"chart-example1.internal","pathType":"ImplementationSpecific","paths":["/example1"]},{"host":"chart-example2.internal","pathType":"ImplementationSpecific","paths":["/example2","/example2/healthz"]}],"tls":[]},"initContainers":[],"istio":{"enable":false,"gateway":{"annotations":{},"enabled":false,"host":"example.com","labels":{},"tls":{"enabled":false,"secretName":"secret-name"}},"virtualService":{"annotations":{},"enabled":false,"gateways":[],"hosts":[],"http":[{"corsPolicy":{},"headers":{},"match":[{"uri":{"prefix":"/v1"}},{"uri":{"prefix":"/v2"}}],"retries":{"attempts":2,"perTryTimeout":"3s"},"rewriteUri":"/","route":[{"destination":{"host":"service1","port":80}}],"timeout":"12s"},{"route":[{"destination":{"host":"service2"}}]}],"labels":{}}},"kedaAutoscaling":{"advanced":{},"authenticationRef":{},"cooldownPeriod":300,"enabled":false,"envSourceContainerName":"","fallback":{},"idleReplicaCount":0,"maxReplicaCount":2,"minReplicaCount":1,"pollingInterval":30,"triggerAuthentication":{"enabled":false,"name":"","spec":{}},"triggers":[]},"nodeSelector":{},"orchestrator.deploymant.algo":1,"pauseForSecondsBeforeSwitchActive":30,"podAnnotations":{},"podDisruptionBudget":{},"podExtraSpecs":{},"podLabels":{},"podSecurityContext":{},"prometheus":{"release":"monitoring"},"prometheusRule":{"additionalLabels":{},"enabled":false,"namespace":""},"rawYaml":[],"replicaCount":1,"resources":{"limits":{"cpu":"0.05","memory":"50Mi"},"requests":{"cpu":"0.01","memory":"10Mi"}},"rolloutAnnotations":{},"rolloutLabels":{},"secret":{"data":{},"enabled":false},"server":{"deployment":{"image":"","image_tag":"1-95af053"}},"service":{"annotations":{},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":false,"name":""},"servicemonitor":{"additionalLabels":{}},"tolerations":[],"topologySpreadConstraints":[],"volumeMounts":[],"volumes":[],"waitForSecondsBeforeScalingDown":30}` - variables, err := templateParser.ExtractVariables(sampleTemplate) + variables, err := templateParser.ExtractVariables(sampleTemplate, JsonVariableTemplate) assert.Nil(t, err) assert.Equal(t, 3, len(variables)) assert.Equal(t, "envoyPort", variables[0]) @@ -47,9 +49,11 @@ func TestVariableTemplateParserImpl_ExtractVariables(t *testing.T) { } func TestVariableTemplateParserImpl_ParseTemplate(t *testing.T) { + t.Setenv("SCOPED_VARIABLE_ENABLED", "true") logger, err := util.NewSugardLogger() assert.Nil(t, err) - templateParser := NewVariableTemplateParserImpl(logger) + templateParser, err := NewVariableTemplateParserImpl(logger) + assert.Nil(t, err) t.Run("parse template", func(t *testing.T) { scopedVariables := []*models.ScopedVariableData{{VariableName: "container-port-number-new", VariableValue: &models.VariableValue{Value: "1800"}}} parserResponse := templateParser.ParseTemplate(VariableParserRequest{TemplateType: JsonVariableTemplate, Template: JsonWithIntParam, Variables: scopedVariables}) diff --git a/pkg/variables/repository/VariableEntityMappingRepository.go b/pkg/variables/repository/VariableEntityMappingRepository.go index ddbf2cb0c1..de2ce409c1 100644 --- a/pkg/variables/repository/VariableEntityMappingRepository.go +++ b/pkg/variables/repository/VariableEntityMappingRepository.go @@ -27,11 +27,28 @@ import ( type VariableEntityMappingRepository interface { sql.TransactionWrapper GetVariablesForEntities(entities []Entity) ([]*VariableEntityMapping, error) + GetLiveUsagesForVariableNames(variableNames []string) ([]*VariableUsageRow, error) SaveVariableEntityMappings(tx *pg.Tx, mappings []*VariableEntityMapping) error DeleteAllVariablesForEntities(tx *pg.Tx, entities []Entity, userId int32) error DeleteVariablesForEntity(tx *pg.Tx, variableIDs []string, entity Entity, userId int32) error } +// VariableUsageRow is one live reference of a variable, resolved to the app/env/pipeline it belongs to. +// A mapping row is considered live only when the entity that holds the reference is still the +// currently-served config (latest chart / latest env override / non-deleted pipeline or config map) +// and its app/environment is active. +type VariableUsageRow struct { + VariableName string `sql:"variable_name"` + EntityType EntityType `sql:"entity_type"` + EntityId int `sql:"entity_id"` + AppId int `sql:"app_id"` + AppName string `sql:"app_name"` + EnvId int `sql:"env_id"` + EnvName string `sql:"env_name"` + PipelineName string `sql:"pipeline_name"` + StageType string `sql:"stage_type"` +} + func NewVariableEntityMappingRepository(logger *zap.SugaredLogger, dbConnection *pg.DB, TransactionUtilImpl *sql.TransactionUtilImpl) *VariableEntityMappingRepositoryImpl { return &VariableEntityMappingRepositoryImpl{ logger: logger, @@ -73,6 +90,71 @@ func (impl *VariableEntityMappingRepositoryImpl) GetVariablesForEntities(entitie return mappings, nil } +const liveVariableUsageQuery = ` +SELECT vem.variable_name, vem.entity_type, vem.entity_id, + a.id AS app_id, COALESCE(NULLIF(a.display_name, ''), a.app_name) AS app_name, + NULL::int AS env_id, NULL::varchar AS env_name, + NULL::varchar AS pipeline_name, NULL::varchar AS stage_type +FROM variable_entity_mapping vem +INNER JOIN charts c ON c.id = vem.entity_id AND c.active = true AND c.latest = true +INNER JOIN app a ON a.id = c.app_id AND a.active = true +WHERE vem.entity_type = 1 AND vem.is_deleted = false AND vem.variable_name IN (?) +UNION ALL +SELECT vem.variable_name, vem.entity_type, vem.entity_id, + a.id, COALESCE(NULLIF(a.display_name, ''), a.app_name), + e.id, e.environment_name, NULL, NULL +FROM variable_entity_mapping vem +INNER JOIN chart_env_config_override ceco ON ceco.id = vem.entity_id AND ceco.active = true AND ceco.latest = true +INNER JOIN charts c ON c.id = ceco.chart_id +INNER JOIN app a ON a.id = c.app_id AND a.active = true +INNER JOIN environment e ON e.id = ceco.target_environment AND e.active = true +WHERE vem.entity_type = 2 AND vem.is_deleted = false AND vem.variable_name IN (?) +UNION ALL +SELECT vem.variable_name, vem.entity_type, vem.entity_id, + a.id, COALESCE(NULLIF(a.display_name, ''), a.app_name), + e.id, e.environment_name, + COALESCE(cp.name, p.pipeline_name), ps.type::varchar +FROM variable_entity_mapping vem +INNER JOIN pipeline_stage ps ON ps.id = vem.entity_id AND ps.deleted = false +LEFT JOIN ci_pipeline cp ON cp.id = ps.ci_pipeline_id AND cp.deleted = false +LEFT JOIN pipeline p ON p.id = ps.cd_pipeline_id AND p.deleted = false +LEFT JOIN environment e ON e.id = p.environment_id +INNER JOIN app a ON a.id = COALESCE(cp.app_id, p.app_id) AND a.active = true +WHERE vem.entity_type = 3 AND vem.is_deleted = false AND vem.variable_name IN (?) + AND (cp.id IS NOT NULL OR p.id IS NOT NULL) +UNION ALL +SELECT vem.variable_name, vem.entity_type, vem.entity_id, + a.id, COALESCE(NULLIF(a.display_name, ''), a.app_name), + NULL, NULL, NULL, NULL +FROM variable_entity_mapping vem +INNER JOIN config_map_app_level cmal ON cmal.id = vem.entity_id +INNER JOIN app a ON a.id = cmal.app_id AND a.active = true +WHERE vem.entity_type IN (4, 6) AND vem.is_deleted = false AND vem.variable_name IN (?) +UNION ALL +SELECT vem.variable_name, vem.entity_type, vem.entity_id, + a.id, COALESCE(NULLIF(a.display_name, ''), a.app_name), + e.id, e.environment_name, NULL, NULL +FROM variable_entity_mapping vem +INNER JOIN config_map_env_level cmel ON cmel.id = vem.entity_id AND cmel.deleted = false +INNER JOIN app a ON a.id = cmel.app_id AND a.active = true +INNER JOIN environment e ON e.id = cmel.environment_id AND e.active = true +WHERE vem.entity_type IN (5, 7) AND vem.is_deleted = false AND vem.variable_name IN (?) +` + +func (impl *VariableEntityMappingRepositoryImpl) GetLiveUsagesForVariableNames(variableNames []string) ([]*VariableUsageRow, error) { + usages := make([]*VariableUsageRow, 0) + if len(variableNames) == 0 { + return usages, nil + } + names := pg.In(variableNames) + _, err := impl.dbConnection.Query(&usages, liveVariableUsageQuery, names, names, names, names, names) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("err in getting live usages for variable names", "variableNames", variableNames, "err", err) + return nil, err + } + return usages, nil +} + func (impl *VariableEntityMappingRepositoryImpl) DeleteVariablesForEntity(tx *pg.Tx, variableNames []string, entity Entity, userId int32) error { _, err := tx.Model((*VariableEntityMapping)(nil)). diff --git a/pkg/variables/repository/VariableEntityMappingRepository_test.go b/pkg/variables/repository/VariableEntityMappingRepository_test.go index c89a3a1251..a49aa3cbc5 100644 --- a/pkg/variables/repository/VariableEntityMappingRepository_test.go +++ b/pkg/variables/repository/VariableEntityMappingRepository_test.go @@ -32,14 +32,19 @@ func getDbConnAndLoggerService(t *testing.T) (*zap.SugaredLogger, *pg.DB) { logger, err := utils.NewSugardLogger() assert.Nil(t, err) dbConnection, err := sql.NewDbConnection(cfg, logger) - assert.Nil(t, err) + if err != nil || dbConnection == nil { + t.Skipf("skipping DB test, database not reachable: %v", err) + } + if _, err = dbConnection.Exec("SELECT 1"); err != nil { + t.Skipf("skipping DB test, database not reachable: %v", err) + } return logger, dbConnection } func getVariableEntityMappingRepositoryImpl(t *testing.T) *VariableEntityMappingRepositoryImpl { logger, dbConnection := getDbConnAndLoggerService(t) - return NewVariableEntityMappingRepository(logger, dbConnection) + return NewVariableEntityMappingRepository(logger, dbConnection, sql.NewTransactionUtilImpl(dbConnection)) } func TestVariableEntityMappingRepositoryImpl_DeleteAllVariablesForEntities(t *testing.T) { @@ -68,7 +73,7 @@ func TestVariableEntityMappingRepositoryImpl_DeleteAllVariablesForEntities(t *te for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { impl := getVariableEntityMappingRepositoryImpl(t) - if err := impl.DeleteAllVariablesForEntities(tt.args.entities, tt.args.userId); (err != nil) != tt.wantErr { + if err := impl.DeleteAllVariablesForEntities(nil, tt.args.entities, tt.args.userId); (err != nil) != tt.wantErr { t.Errorf("DeleteAllVariablesForEntities() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -224,3 +229,18 @@ func TestVariableEntityMappingRepositoryImpl_SaveVariableEntityMappings(t *testi }) } } + +func TestVariableEntityMappingRepositoryImpl_GetLiveUsagesForVariableNames(t *testing.T) { + t.Run("empty input returns empty without querying", func(t *testing.T) { + impl := getVariableEntityMappingRepositoryImpl(t) + usages, err := impl.GetLiveUsagesForVariableNames(nil) + assert.Nil(t, err) + assert.Empty(t, usages) + }) + t.Run("query executes and returns empty for unknown variables", func(t *testing.T) { + impl := getVariableEntityMappingRepositoryImpl(t) + usages, err := impl.GetLiveUsagesForVariableNames([]string{"UNKNOWN_VARIABLE_FOR_TEST_1", "UNKNOWN_VARIABLE_FOR_TEST_2"}) + assert.Nil(t, err) + assert.Empty(t, usages) + }) +} diff --git a/scripts/sql/36504600_variable_entity_mapping_index.down.sql b/scripts/sql/36504600_variable_entity_mapping_index.down.sql new file mode 100644 index 0000000000..b816bb9d2f --- /dev/null +++ b/scripts/sql/36504600_variable_entity_mapping_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_variable_entity_mapping_variable_name; diff --git a/scripts/sql/36504600_variable_entity_mapping_index.up.sql b/scripts/sql/36504600_variable_entity_mapping_index.up.sql new file mode 100644 index 0000000000..96633dc6b6 --- /dev/null +++ b/scripts/sql/36504600_variable_entity_mapping_index.up.sql @@ -0,0 +1,4 @@ +-- index for looking up live usages of a scoped variable by name (variable deletion protection) +CREATE INDEX IF NOT EXISTS idx_variable_entity_mapping_variable_name + ON variable_entity_mapping (variable_name) + WHERE is_deleted = false; diff --git a/wire_gen.go b/wire_gen.go index e3518a51bb..1918fe255e 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -533,12 +533,12 @@ func InitializeApp() (*App, error) { if err != nil { return nil, err } - scopedVariableServiceImpl, err := variables.NewScopedVariableServiceImpl(sugaredLogger, scopedVariableRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, devtronResourceSearchableKeyServiceImpl, clusterRepositoryImpl, qualifierMappingServiceImpl, runnable) + variableEntityMappingRepositoryImpl := repository14.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) + variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) + scopedVariableServiceImpl, err := variables.NewScopedVariableServiceImpl(sugaredLogger, scopedVariableRepositoryImpl, appRepositoryImpl, environmentRepositoryImpl, devtronResourceSearchableKeyServiceImpl, clusterRepositoryImpl, qualifierMappingServiceImpl, variableEntityMappingServiceImpl, runnable) if err != nil { return nil, err } - variableEntityMappingRepositoryImpl := repository14.NewVariableEntityMappingRepository(sugaredLogger, db, transactionUtilImpl) - variableEntityMappingServiceImpl := variables.NewVariableEntityMappingServiceImpl(variableEntityMappingRepositoryImpl, sugaredLogger) variableSnapshotHistoryRepositoryImpl := repository14.NewVariableSnapshotHistoryRepository(sugaredLogger, db) variableSnapshotHistoryServiceImpl := variables.NewVariableSnapshotHistoryServiceImpl(variableSnapshotHistoryRepositoryImpl, sugaredLogger) variableTemplateParserImpl, err := parsers.NewVariableTemplateParserImpl(sugaredLogger) From 2d0822b4d54621ab80631e5701de24106d848996 Mon Sep 17 00:00:00 2001 From: bhupesh-devtron Date: Thu, 3 Sep 2026 14:18:35 +0530 Subject: [PATCH 05/12] perf: add ci_workflow status index (#7017) --- ...d_ci_workflow_active_status_index.down.sql | 1 + ...add_ci_workflow_active_status_index.up.sql | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 scripts/sql/36504600_add_ci_workflow_active_status_index.down.sql create mode 100644 scripts/sql/36504600_add_ci_workflow_active_status_index.up.sql diff --git a/scripts/sql/36504600_add_ci_workflow_active_status_index.down.sql b/scripts/sql/36504600_add_ci_workflow_active_status_index.down.sql new file mode 100644 index 0000000000..11441ac52b --- /dev/null +++ b/scripts/sql/36504600_add_ci_workflow_active_status_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_ci_workflow_active_status; diff --git a/scripts/sql/36504600_add_ci_workflow_active_status_index.up.sql b/scripts/sql/36504600_add_ci_workflow_active_status_index.up.sql new file mode 100644 index 0000000000..a65c5046df --- /dev/null +++ b/scripts/sql/36504600_add_ci_workflow_active_status_index.up.sql @@ -0,0 +1,24 @@ +-- Supports CiWorkflowRepositoryImpl.FindByStatusesIn, called every 5 minutes by +-- the CI workflow status cron (UpdateCiWorkflowStatusFailure) to find builds +-- stuck in Starting/Running so they can be timed out. With no index on status, +-- this does a full sequential scan of ci_workflow (tens of millions of rows in +-- production) on every run. +-- +-- Partial index scoped to only the two statuses this query filters on, so it +-- stays small (proportional to active build count, not total table size) and +-- cheap to maintain on INSERT/UPDATE, regardless of how large ci_workflow grows. +-- +-- Note: CREATE INDEX CONCURRENTLY cannot run inside a transaction (golang-migrate +-- wraps each migration in a transaction). We use plain CREATE INDEX IF NOT EXISTS +-- here so the migration is idempotent. For large existing ci_workflow tables, +-- operators SHOULD create this index manually with CONCURRENTLY BEFORE deploying +-- this release, to avoid blocking writes during pod startup (~2 minutes to build +-- on a table with 40 million rows, benchmarked locally): +-- +-- CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_ci_workflow_active_status +-- ON ci_workflow (status) WHERE status IN ('Starting', 'Running'); +-- +-- Once the index exists, this migration is a no-op (IF NOT EXISTS). + +CREATE INDEX IF NOT EXISTS idx_ci_workflow_active_status + ON ci_workflow (status) WHERE status IN ('Starting', 'Running'); From c8688762c0dc0d4ab7c0ccff424b2d55ce73168f Mon Sep 17 00:00:00 2001 From: Prakash Kumar Date: Wed, 9 Sep 2026 18:40:17 +0530 Subject: [PATCH 06/12] scope variable multi apps, envs fixes --- .../ScopedVariableRestHandler.go | 7 +- pkg/variables/utils/transformer.go | 142 ++++++++++++++++-- pkg/variables/utils/transformer_test.go | 84 +++++++++++ 3 files changed, 223 insertions(+), 10 deletions(-) create mode 100644 pkg/variables/utils/transformer_test.go diff --git a/api/restHandler/scopedVariable/ScopedVariableRestHandler.go b/api/restHandler/scopedVariable/ScopedVariableRestHandler.go index 049d38542b..986cefaa0c 100644 --- a/api/restHandler/scopedVariable/ScopedVariableRestHandler.go +++ b/api/restHandler/scopedVariable/ScopedVariableRestHandler.go @@ -96,7 +96,12 @@ func (handler *ScopedVariableRestHandlerImpl) CreateVariables(w http.ResponseWri return } - payload := utils.ManifestToPayload(request.Manifest, userId) + payload, err := utils.ManifestToPayload(request.Manifest, userId) + if err != nil { + handler.logger.Errorw("invalid manifest in CreateVariables", "err", err) + common.WriteJsonResp(w, err, nil, http.StatusNotAcceptable) + return + } // not logging bean object as it contains sensitive data handler.logger.Infow("request payload received for variables") diff --git a/pkg/variables/utils/transformer.go b/pkg/variables/utils/transformer.go index eb29aea323..d05ad7de2d 100644 --- a/pkg/variables/utils/transformer.go +++ b/pkg/variables/utils/transformer.go @@ -17,25 +17,98 @@ package utils import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/devtron-labs/devtron/pkg/variables/helper" "github.com/devtron-labs/devtron/pkg/variables/models" ) -func ManifestToPayload(manifest models.ScopedVariableManifest, userId int32) models.Payload { +const groupedNameSeparator = "," +const groupedNameJoiner = ", " + +// GetGroupedIdentifierForCategory returns the singular selector key of a category together with its +// grouped (plural) form, e.g. Env -> (EnvName, EnvNames). Grouped selectors are supported only for +// categories with exactly one selector key. In OSS only the Global category (no selectors) exists, +// so this always returns ok=false here; the logic is kept in sync with enterprise. +func GetGroupedIdentifierForCategory(category models.AttributeType) (singular models.IdentifierType, grouped models.IdentifierType, ok bool) { + identifierTypes := helper.GetIdentifierTypeFromAttributeType(category) + if len(identifierTypes) != 1 { + return "", "", false + } + singular = identifierTypes[0] + return singular, models.IdentifierType(string(singular) + "s"), true +} + +// ExpandAttributeSelectors turns one selector map that may carry a grouped key +// (e.g. EnvNames: "env1, env2") into one selector map per name carrying the singular key +// (EnvName: "env1"). A map without a grouped key is returned unchanged. +func ExpandAttributeSelectors(category models.AttributeType, attributeSelectors map[models.IdentifierType]string) ([]map[models.IdentifierType]string, error) { + singular, grouped, ok := GetGroupedIdentifierForCategory(category) + if !ok { + return []map[models.IdentifierType]string{attributeSelectors}, nil + } + groupedValue, hasGrouped := attributeSelectors[grouped] + if !hasGrouped { + return []map[models.IdentifierType]string{attributeSelectors}, nil + } + if _, hasSingular := attributeSelectors[singular]; hasSingular { + return nil, fmt.Errorf("selector for category %s cannot use both %s and %s", category, singular, grouped) + } + names := make([]string, 0) + seen := make(map[string]bool) + for _, name := range strings.Split(groupedValue, groupedNameSeparator) { + name = strings.TrimSpace(name) + if name == "" { + return nil, fmt.Errorf("%s for category %s contains an empty name", grouped, category) + } + if seen[name] { + return nil, fmt.Errorf("%s for category %s contains duplicate name %q", grouped, category, name) + } + seen[name] = true + names = append(names, name) + } + expanded := make([]map[models.IdentifierType]string, 0, len(names)) + for _, name := range names { + selectors := make(map[models.IdentifierType]string, len(attributeSelectors)) + for key, value := range attributeSelectors { + if key != grouped { + selectors[key] = value + } + } + selectors[singular] = name + expanded = append(expanded, selectors) + } + return expanded, nil +} + +func ManifestToPayload(manifest models.ScopedVariableManifest, userId int32) (models.Payload, error) { variableList := make([]*models.Variables, 0) for _, spec := range manifest.Spec { attributes := make([]models.AttributeValue, 0) for _, value := range spec.Values { - attribute := models.AttributeValue{ - VariableValue: models.VariableValue{Value: value.Value}, - AttributeType: models.Global, - } - + selectorsList := []map[models.IdentifierType]string{nil} if value.Selectors != nil && value.Selectors.AttributeSelectors != nil { - attribute.AttributeParams = value.Selectors.AttributeSelectors + expandedSelectors, err := ExpandAttributeSelectors(value.Category, value.Selectors.AttributeSelectors) + if err != nil { + return models.Payload{}, models.ValidationError{Err: err} + } + selectorsList = expandedSelectors + } + for _, selectors := range selectorsList { + attribute := models.AttributeValue{ + VariableValue: models.VariableValue{Value: value.Value}, + AttributeType: value.Category, + } + if selectors != nil { + attribute.AttributeParams = selectors + } + attributes = append(attributes, attribute) } - attributes = append(attributes, attribute) } variable := models.Variables{ Definition: models.Definition{ @@ -56,7 +129,32 @@ func ManifestToPayload(manifest models.ScopedVariableManifest, userId int32) mod Variables: variableList, UserId: userId, } - return payload + return payload, nil +} + +// groupingValueKey renders a value into a stable, type-aware key so that only entries carrying the +// same value are grouped (json.Marshal keeps 5 and "5" distinct and sorts map keys). +func groupingValueKey(value interface{}) string { + marshaled, err := json.Marshal(value) + if err != nil { + return fmt.Sprintf("%v", value) + } + return string(marshaled) +} + +// isGroupableAttribute reports whether an attribute entry may be merged with others of the same +// category and value: single-selector category, exactly the singular key present, and a name that +// itself contains no separator. +func isGroupableAttribute(attribute models.AttributeValue) (name string, ok bool) { + singular, _, ok := GetGroupedIdentifierForCategory(attribute.AttributeType) + if !ok || len(attribute.AttributeParams) != 1 { + return "", false + } + name, hasSingular := attribute.AttributeParams[singular] + if !hasSingular || strings.Contains(name, groupedNameSeparator) { + return "", false + } + return name, true } func PayloadToManifest(payload models.Payload) models.ScopedVariableManifest { @@ -73,7 +171,20 @@ func PayloadToManifest(payload models.Payload) models.ScopedVariableManifest { Values: make([]models.VariableValueSpec, 0), IsSensitive: variable.Definition.VarType.IsTypeSensitive(), } + // entries of the same category carrying the same value are folded into one grouped entry + groupKeyToValueIndex := make(map[string]int) + groupNames := make(map[int][]string) for _, attribute := range variable.AttributeValues { + name, groupable := isGroupableAttribute(attribute) + if groupable { + groupKey := string(attribute.AttributeType) + "\x00" + groupingValueKey(attribute.VariableValue.Value) + if valueIndex, found := groupKeyToValueIndex[groupKey]; found { + groupNames[valueIndex] = append(groupNames[valueIndex], name) + continue + } + groupKeyToValueIndex[groupKey] = len(spec.Values) + groupNames[len(spec.Values)] = []string{name} + } valueSpec := models.VariableValueSpec{ Value: attribute.VariableValue.Value, Category: attribute.AttributeType, @@ -83,6 +194,19 @@ func PayloadToManifest(payload models.Payload) models.ScopedVariableManifest { } spec.Values = append(spec.Values, valueSpec) } + for valueIndex, names := range groupNames { + if len(names) < 2 { + continue + } + _, grouped, ok := GetGroupedIdentifierForCategory(spec.Values[valueIndex].Category) + if !ok { + continue + } + sort.Strings(names) + spec.Values[valueIndex].Selectors = &models.Selector{AttributeSelectors: map[models.IdentifierType]string{ + grouped: strings.Join(names, groupedNameJoiner), + }} + } manifest.Spec = append(manifest.Spec, spec) } return manifest diff --git a/pkg/variables/utils/transformer_test.go b/pkg/variables/utils/transformer_test.go new file mode 100644 index 0000000000..ab5cbb68f7 --- /dev/null +++ b/pkg/variables/utils/transformer_test.go @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "testing" + + "github.com/devtron-labs/devtron/pkg/variables/models" + "github.com/stretchr/testify/assert" +) + +// OSS supports only the Global category (no attribute selectors), so grouped selector keys +// (EnvNames, ...) can never trigger expansion here. These tests pin that dormancy and the +// unchanged round-trip behaviour; the expansion logic itself is exercised in enterprise. + +func TestGetGroupedIdentifierForCategory_AlwaysFalseInOSS(t *testing.T) { + for _, category := range []models.AttributeType{models.Global, "Env", "Application", "Cluster"} { + _, _, ok := GetGroupedIdentifierForCategory(category) + assert.False(t, ok, string(category)) + } +} + +func TestManifestPayloadRoundTrip_GlobalOnly(t *testing.T) { + original := models.ScopedVariableManifest{ + ApiVersion: "devtron.ai/v1beta1", + Kind: "Variable", + Spec: []models.VariableSpec{ + { + Name: "VAR1", + Notes: "notes", + ShortDescription: "short", + Values: []models.VariableValueSpec{ + {Category: models.Global, Value: "value1"}, + }, + }, + }, + } + payload, err := ManifestToPayload(original, 3) + assert.Nil(t, err) + assert.Equal(t, int32(3), payload.UserId) + assert.Equal(t, original.Spec, PayloadToManifest(payload).Spec) +} + +func TestManifestToPayload_SelectorsPassThroughUnchangedInOSS(t *testing.T) { + manifest := models.ScopedVariableManifest{ + ApiVersion: "devtron.ai/v1beta1", + Kind: "Variable", + Spec: []models.VariableSpec{ + { + Name: "VAR1", + Values: []models.VariableValueSpec{ + { + Category: models.Global, + Value: "v", + Selectors: &models.Selector{AttributeSelectors: map[models.IdentifierType]string{ + "EnvNames": "env1, env2", + }}, + }, + }, + }, + }, + } + payload, err := ManifestToPayload(manifest, 1) + + assert.Nil(t, err) + attributes := payload.Variables[0].AttributeValues + assert.Len(t, attributes, 1) + // no expansion in OSS; the invalid key is left for payload validation to reject + assert.Equal(t, "env1, env2", attributes[0].AttributeParams["EnvNames"]) +} From ea5174ed9101a9b6b98f3a327331738fc8d1e05e Mon Sep 17 00:00:00 2001 From: Asutosh Das Date: Mon, 3 Aug 2026 16:08:09 +0530 Subject: [PATCH 07/12] feat: foreground (detach) delete of CD pipelines Cherry-picked from devtron-enterprise #3373. * add FOREGROUND_DELETE_CD_PIPELINE env config * add nullable foregroundDelete query param + resolution against env default * skip deployed resource deletion (helm/argocd/flux) on foreground CD pipeline delete * honor foregroundDelete in async CD pipeline delete event * validate conflicting force/cascade/foregroundDelete options in PatchCdPipeline * expose foregroundDeleteCdPipeline in GET /environment-variables --- .../DeploymentPipelineRestHandler.go | 41 +++++++++++++++---- env_gen.json | 2 +- env_gen.md | 1 + pkg/bean/app.go | 10 +++++ pkg/bean/app_test.go | 28 +++++++++++++ pkg/bulkAction/service/BulkUpdateService.go | 5 ++- pkg/commonService/CommonBaseService.go | 1 + pkg/commonService/bean_ent.go | 3 +- ...eployedApplicationEventProcessorService.go | 2 +- .../in/WorkflowEventProcessorService.go | 15 ++++--- .../out/PipelineConfigEventPublishService.go | 9 ++-- pkg/eventProcessor/out/bean/bean.go | 5 ++- .../DeploymentPipelineConfigService.go | 22 ++++++---- util/GlobalConfig.go | 1 + wire_gen.go | 2 +- 15 files changed, 113 insertions(+), 34 deletions(-) create mode 100644 pkg/bean/app_test.go diff --git a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go index 5260593e56..f0e82c9591 100644 --- a/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/DeploymentPipelineRestHandler.go @@ -296,32 +296,55 @@ func (handler *PipelineConfigRestHandlerImpl) PatchCdPipeline(w http.ResponseWri } v := r.URL.Query() - forceDelete := false - cascadeDelete := true force := v.Get("force") cascade := v.Get("cascade") - if len(force) > 0 && len(cascade) > 0 { - handler.Logger.Errorw("request err, PatchCdPipeline", "err", fmt.Errorf("cannot perform both cascade and force delete"), "payload", cdPipeline) + foreground := v.Get("foregroundDelete") + if len(force) > 0 && len(cascade) > 0 && len(foreground) > 0 { + err = fmt.Errorf("cannot perform force, cascade and foreground delete together") + handler.Logger.Errorw("request err, PatchCdPipeline", "err", err, "payload", cdPipeline) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } else if len(force) > 0 && len(cascade) > 0 { + err = fmt.Errorf("cannot perform both cascade and force delete") + handler.Logger.Errorw("request err, PatchCdPipeline", "err", err, "payload", cdPipeline) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } else if len(foreground) > 0 && len(cascade) > 0 { + err = fmt.Errorf("cannot perform both cascade and foreground delete") + handler.Logger.Errorw("request err, PatchCdPipeline", "err", err, "payload", cdPipeline) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } else if len(foreground) > 0 && len(force) > 0 { + err = fmt.Errorf("cannot perform both force and foreground delete") + handler.Logger.Errorw("request err, PatchCdPipeline", "err", err, "payload", cdPipeline) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - if len(force) > 0 { - forceDelete, err = strconv.ParseBool(force) + if len(foreground) > 0 { + foregroundDelete, err := strconv.ParseBool(foreground) + if err != nil { + handler.Logger.Errorw("request err, PatchCdPipeline", "err", err, "payload", cdPipeline) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } + cdPipeline.ForegroundDelete = &foregroundDelete + } else if len(force) > 0 { + forceDelete, err := strconv.ParseBool(force) if err != nil { handler.Logger.Errorw("request err, PatchCdPipeline", "err", err, "payload", cdPipeline) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + cdPipeline.ForceDelete = forceDelete } else if len(cascade) > 0 { - cascadeDelete, err = strconv.ParseBool(cascade) + cascadeDelete, err := strconv.ParseBool(cascade) if err != nil { handler.Logger.Errorw("request err, PatchCdPipeline", "err", err, "payload", cdPipeline) common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } + cdPipeline.NonCascadeDelete = !cascadeDelete } - cdPipeline.ForceDelete = forceDelete - cdPipeline.NonCascadeDelete = !cascadeDelete handler.Logger.Infow("request payload, PatchCdPipeline", "payload", cdPipeline) err = handler.validator.StructPartial(cdPipeline, "AppId", "Action") if err == nil { diff --git a/env_gen.json b/env_gen.json index 15fb4e0dbe..c7093ea34a 100644 --- a/env_gen.json +++ b/env_gen.json @@ -1 +1 @@ -[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"CD_FLUX_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status for flux cd pipeline","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status ","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time for CD pipeline status","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"Timeout for CD pipeline to get healthy","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses.","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"Context timeout for gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"Context timeout for no gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_FLUX_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable flux application services","Example":"","Deprecated":"false"},{"Env":"FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking flux app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops.","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"Time to mark a pipeline degraded if not healthy in defined time","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"Count for devtron application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"Count for external helm application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"To set the history limit for the helm app being deployed through devtron","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"},{"Env":"VALIDATE_EXT_APP_CHART_TYPE","EnvType":"bool","EnvValue":"false","EnvDescription":"validate external flux app chart","Example":"","Deprecated":"false"}]},{"Category":"CI_BUILDX","Fields":[{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async container image cache export","Example":"","Deprecated":"false"},{"Env":"BUILDX_BUILDER_POD_WAIT_DURATION_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"Timeout in seconds to wait for buildx k8s driver builder pods to be ready (initial startup and after spot interruption)","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"To set build cache mode to minimum in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_INTERRUPTION_MAX_RETRY","EnvType":"int","EnvValue":"3","EnvDescription":"Maximum number of retries for buildx builder interruption","Example":"","Deprecated":"false"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"If blob storage is being used of azure then pass the secret key to access the bucket","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"Account name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"Cache bucket name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"Log bucket for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"Azure gateway connection allows insecure if true","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"Sent to CI runner for blob","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"Used to store, download logs of ci workflow, artifact","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"GCP cred json for GCS blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"Blob storage provider name(AWS/GCP/Azure)","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"S3 access key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable buctet versioning for blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"S3 endpoint URL for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"To use insecure s3 endpoint","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Secret key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"Path for the buildx cache","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"To enable the k8s driver and pass args for k8s driver in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"This is the time that the pods of ci/pre-cd/post-cd live after completion state.","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"Cache limit.","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for Pre/Post cd ","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"Toleration key for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"Toleration value for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for Pre/Post CD(AWF,System)","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"Service account to be used in Pre/Post CD pod","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for CI","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CI","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"Ignoring docker cache ","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for build logs","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Toleration key for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"Toleration value for CI","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value)","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"additional volume mount data for CI and JOB","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for CI(AWF,System)","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"Key location for artifacts being created","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"Bucket prefix for build logs","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"Bucket name for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"Build Cache bucket region","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"Bucket prefix for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"Namespace for devtron stack","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"Timeout for Pre/Post-Cd to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"To pass the ci-runner image","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"Default architecture for buildx","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"Path to store cache of docker build (/var/lib/docker-\u003e for legacy docker build, /var/lib/devtron-\u003e for buildx)","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable build context in Devtron.","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"This is an array of strings used when submitting a workflow for pre or post-CD execution. If the ","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"External CI API secret.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"External CI payload with project details.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci.","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"Ignore CM/CS in CI-pipeline as Job","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"push artifact(image) in ci retry count ","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"image retry interval takes value in seconds","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"Image-scanner micro-service URL","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"Max retry count for image-scanning","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay for the image-scaning to start","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow.","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time pre/post-cd-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time CI-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"Orchestrator micro-service URL ","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"Orchestrator token","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"Cache path for Pre CI tasks","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable showing the args passed for CI in build logs","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"To skip cache Push/Pull for ci job","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"By disabling this ECR repo won't get created if it's not available on ECR from build configuration","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"this is the time given to workflow pods to shutdown. (grace full termination time)","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 query for listing artifacts","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post cd","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post ci","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable buildx feature globally","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner]","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location.","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"To use the same tag in container image as that of git tag","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"Workflow controller instance ID.","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"flag is used to configure how Docker caches are handled during a CI/CD ","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"For the app sync image, this image will be used in app-manual sync job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"To pass the resource of app sync","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"Service account to be used in app sync Job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled all argocd application will have auto sync enabled","Example":"true","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"Delay on retrying the maifest commit the on gitops","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"4","EnvDescription":"Retry count for registering a GitOps repository to ArgoCD","Example":"3","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay (in Seconds) between the retries for registering a GitOps repository to ArgoCD","Example":"5","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go.","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host for the devtron stack","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"Port for pre/post-cd","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"Caching expiration time.","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"For image poll plugin","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"Cron schedule for CI pipeline status","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"Used in git cli opeartion timeout","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_BACKGROUND_REFRESH_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"Enable background refresh of cluster overview cache","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"Enable caching for cluster overview data","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_MAX_PARALLEL_CLUSTERS","EnvType":"int","EnvValue":"15","EnvDescription":"Maximum number of clusters to fetch in parallel during refresh","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_MAX_STALE_DATA_SECONDS","EnvType":"int","EnvValue":"30","EnvDescription":"Maximum age of cached data in seconds before warning","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_REFRESH_INTERVAL_SECONDS","EnvType":"int","EnvValue":"15","EnvDescription":"Background cache refresh interval in seconds","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"Cron schedule for cluster status on resource browser","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of dex secret","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Name of the Devtron Helm release. ","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of the Devtron Helm release","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"Devtron Installation type(EA/Full)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLER_MODULES_PATH","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"Path to devtron installer modules, used to find the helm charts and values files","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLER_RELEASE_PATH","EnvType":"string","EnvValue":"installer.release","EnvDescription":"Path to devtron installer release, used to find the helm charts and values files","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_OPERATOR_BASE_PATH","EnvType":"string","EnvValue":"","EnvDescription":"Base path for devtron operator, used to find the helm charts and values files","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"devtron operator version identifier in helm values yaml","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"dex client id ","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX CSTOREKEY.","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX JWT key. ","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"Dex secret","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"Prefix for ECR repo to be created in does not exist","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of no-gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_LINKED_CI_ARTIFACT_COPY","EnvType":"bool","EnvValue":"false","EnvDescription":"Enable copying artifacts from parent CI pipeline to linked CI pipeline during creation","Example":"","Deprecated":"false"},{"Env":"ENABLE_PASSWORD_ENCRYPTION","EnvType":"bool","EnvValue":"true","EnvDescription":"enable password encryption","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"ephemeral containers support version regex that is compared with k8sServerVersion","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"Notifier service url","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"checks for any nil pointer in wire.go","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"To expose CI metrics","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"restart workload retrieval batch size ","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"restart workload retrieval pool size","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"By enabling this no one can disable image scaning on ci-pipeline from UI","Example":"","Deprecated":"false"},{"Env":"GITHUB_ORG_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITHUB_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITHUB_USERNAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for Gitops repo being creation for argocd application","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace for grafana","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"Org ID for grafana for application metrics","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"Password for grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"Port for grafana micro-service","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"Username for grafana ","Example":"","Deprecated":"false"},{"Env":"HIDE_API_TOKENS","EnvType":"bool","EnvValue":"false","EnvDescription":"Boolean flag for should the api tokens generated be hidden from the UI","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"Flag to hide the hard delete option in the image tagging service","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"flag for ignoring auth check in autocomplete apis.","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"namespace where Custom Resource Definitions get installed","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"Devtron installer CRD group name, partially deprecated.","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"Devtron installer CRD resource name, partially deprecated","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"version of the CRDs. default is v1alpha1","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"JWT expiration time.","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Lens microservice timeout.","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"Lens micro-service URL","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LINKED_CI_ARTIFACT_COPY_LIMIT","EnvType":"int","EnvValue":"10","EnvDescription":"Maximum number of artifacts to copy from parent CI pipeline to linked CI pipeline","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"Enables a different logger theme.","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"max no of cluster terminal pods can be created by an user","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"Modules list and meta info will be fetched from this server, that is central api server of devtron.","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"notification medium","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"Opentelemetry URL ","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"App manual sync job parallel tag processing count.","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository.","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones.","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"Proxy configuration for micro-service to be accessible on orhcestrator ingress","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"To restrict the cluster terminal from user having non-super admin acceess","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable scoped variable option","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"Its a scope format for varialbe name.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"This describe should we handle primitives or not in scoped variable template parsing.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"Regex for scoped variable name that must passed this regex.","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds.","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds.","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"Scoped variable prefix, variable name must have this prefix.","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"Cluster terminal default namespace","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"Timeout for cluster terminal to be inactive","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"this is the time interval at which the status of the cluster terminal pod","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"Timeout for Failed CI build ","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"timeout to compute the urls from services and ingress objects of an application","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 API for listing artifacts in Listing the images in pipeline","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable git cli","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 for RBAC creation","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"This is used to control caching of all the scope variables defined in the system.","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"Scoped variable expression regex","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"If you want to continue using jenkins for CI then please provide this for authentication of requests","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"Name of the argocd CM","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"To pass the argocd namespace","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"Password for the Argocd (deprecated)","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"User name for argocd","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"devtron-gitops-secret","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"this the batch size to control no of above resources can be parsed in one go to determine hibernate status","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Dashboard micro-service URL","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Dashboard micro-service namespace","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"Port for dashboard micro-service","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"Protocol to connect with git-sensor micro-service","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_SERVICE_CONFIG","EnvType":"string","EnvValue":"{\"loadBalancingPolicy\":\"pick_first\"}","EnvDescription":"git-sensor grpc service config","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Timeout for getting response from the git-sensor","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"git-sensor micro-service url ","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"Kubelink micro-service url ","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_SERVICE_CONFIG","EnvType":"string","EnvValue":"{\"loadBalancingPolicy\":\"round_robin\"}","EnvDescription":"kubelink grpc service config","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"Database for casbin","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for read operation in postgres","Example":"","Deprecated":"false"},{"Env":"PG_SSL_MODE","EnvType":"string","EnvValue":"","EnvDescription":"ssl mode for postgres connection","Example":"disable, require, verify-ca, verify-full","Deprecated":"false"},{"Env":"PG_SSL_ROOT_CERT","EnvType":"string","EnvValue":"","EnvDescription":"path to the PEM CA bundle, required for verify-ca/verify-full ssl modes (for AWS RDS use the downloaded global-bundle.pem)","Example":"/etc/devtron/certs/rds-ca-bundle.pem","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for write operation in postgres","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable enforcer cache.","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"Expiration time (in seconds) for enforcer cache. ","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"Maximum batch size for the enforcer.","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable casbin V2 API","Example":"","Deprecated":"false"}]}] \ No newline at end of file +[{"Category":"CD","Fields":[{"Env":"ARGO_APP_MANUAL_SYNC_TIME","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"CD_FLUX_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status for flux cd pipeline","Example":"","Deprecated":"false"},{"Env":"CD_HELM_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time to check the pipeline status ","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_CRON_TIME","EnvType":"string","EnvValue":"*/2 * * * *","EnvDescription":"Cron time for CD pipeline status","Example":"","Deprecated":"false"},{"Env":"CD_PIPELINE_STATUS_TIMEOUT_DURATION","EnvType":"string","EnvValue":"20","EnvDescription":"Timeout for CD pipeline to get healthy","Example":"","Deprecated":"false"},{"Env":"DEPLOY_STATUS_CRON_GET_PIPELINE_DEPLOYED_WITHIN_HOURS","EnvType":"int","EnvValue":"12","EnvDescription":"This flag is used to fetch the deployment status of the application. It retrieves the status of deployments that occurred between 12 hours and 10 minutes prior to the current time. It fetches non-terminal statuses.","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_ARGO_CD_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"1","EnvDescription":"Context timeout for gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"DEVTRON_CHART_INSTALL_REQUEST_TIMEOUT","EnvType":"int","EnvValue":"6","EnvDescription":"Context timeout for no gitops concurrent async deployments","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CD_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable migration of external argocd application to devtron pipeline","Example":"","Deprecated":"false"},{"Env":"FEATURE_MIGRATE_FLUX_APPLICATION_ENABLE","EnvType":"bool","EnvValue":"false","EnvDescription":"enable flux application services","Example":"","Deprecated":"false"},{"Env":"FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking flux app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"FOREGROUND_DELETE_CD_PIPELINE","EnvType":"bool","EnvValue":"false","EnvDescription":"If enabled, deleting a CD pipeline will not delete the underlying deployed resource (helm release/argocd application/flux release) unless the request explicitly overrides it via payload.","Example":"","Deprecated":"false"},{"Env":"HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME","EnvType":"string","EnvValue":"120","EnvDescription":"eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle.","Example":"","Deprecated":"false"},{"Env":"IS_INTERNAL_USE","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops.","Example":"","Deprecated":"false"},{"Env":"MIGRATE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"migrate deployment config data from charts table to deployment_config table","Example":"","Deprecated":"false"},{"Env":"PIPELINE_DEGRADED_TIME","EnvType":"string","EnvValue":"10","EnvDescription":"Time to mark a pipeline degraded if not healthy in defined time","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_DEVTRON_APP","EnvType":"int","EnvValue":"1","EnvDescription":"Count for devtron application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_EXTERNAL_HELM_APP","EnvType":"int","EnvValue":"0","EnvDescription":"Count for external helm application rivision history","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_HELM_APP","EnvType":"int","EnvValue":"1","EnvDescription":"To set the history limit for the helm app being deployed through devtron","Example":"","Deprecated":"false"},{"Env":"REVISION_HISTORY_LIMIT_LINKED_HELM_APP","EnvType":"int","EnvValue":"15","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RUN_HELM_INSTALL_IN_ASYNC_MODE_HELM_APPS","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SHOULD_CHECK_NAMESPACE_ON_CLONE","EnvType":"bool","EnvValue":"false","EnvDescription":"should we check if namespace exists or not while cloning app","Example":"","Deprecated":"false"},{"Env":"USE_DEPLOYMENT_CONFIG_DATA","EnvType":"bool","EnvValue":"false","EnvDescription":"use deployment config data from deployment_config table","Example":"","Deprecated":"true"},{"Env":"VALIDATE_EXT_APP_CHART_TYPE","EnvType":"bool","EnvValue":"false","EnvDescription":"validate external flux app chart","Example":"","Deprecated":"false"}]},{"Category":"CI_BUILDX","Fields":[{"Env":"ASYNC_BUILDX_CACHE_EXPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async container image cache export","Example":"","Deprecated":"false"},{"Env":"BUILDX_BUILDER_POD_WAIT_DURATION_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"Timeout in seconds to wait for buildx k8s driver builder pods to be ready (initial startup and after spot interruption)","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_MODE_MIN","EnvType":"bool","EnvValue":"false","EnvDescription":"To set build cache mode to minimum in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_INTERRUPTION_MAX_RETRY","EnvType":"int","EnvValue":"3","EnvDescription":"Maximum number of retries for buildx builder interruption","Example":"","Deprecated":"false"}]},{"Category":"CI_RUNNER","Fields":[{"Env":"AZURE_ACCOUNT_KEY","EnvType":"string","EnvValue":"","EnvDescription":"If blob storage is being used of azure then pass the secret key to access the bucket","Example":"","Deprecated":"false"},{"Env":"AZURE_ACCOUNT_NAME","EnvType":"string","EnvValue":"","EnvDescription":"Account name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_CACHE","EnvType":"string","EnvValue":"","EnvDescription":"Cache bucket name for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_BLOB_CONTAINER_CI_LOG","EnvType":"string","EnvValue":"","EnvDescription":"Log bucket for azure blob storage","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_CONNECTION_INSECURE","EnvType":"bool","EnvValue":"true","EnvDescription":"Azure gateway connection allows insecure if true","Example":"","Deprecated":"false"},{"Env":"AZURE_GATEWAY_URL","EnvType":"string","EnvValue":"http://devtron-minio.devtroncd:9000","EnvDescription":"Sent to CI runner for blob","Example":"","Deprecated":"false"},{"Env":"BASE_LOG_LOCATION_PATH","EnvType":"string","EnvValue":"/home/devtron/","EnvDescription":"Used to store, download logs of ci workflow, artifact","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_GCP_CREDENTIALS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"GCP cred json for GCS blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_PROVIDER","EnvType":"","EnvValue":"S3","EnvDescription":"Blob storage provider name(AWS/GCP/Azure)","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ACCESS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"S3 access key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_BUCKET_VERSIONED","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable buctet versioning for blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT","EnvType":"string","EnvValue":"","EnvDescription":"S3 endpoint URL for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_ENDPOINT_INSECURE","EnvType":"bool","EnvValue":"false","EnvDescription":"To use insecure s3 endpoint","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_S3_SECRET_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Secret key for s3 blob storage","Example":"","Deprecated":"false"},{"Env":"BUILDX_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/devtron/buildx","EnvDescription":"Path for the buildx cache","Example":"","Deprecated":"false"},{"Env":"BUILDX_K8S_DRIVER_OPTIONS","EnvType":"string","EnvValue":"","EnvDescription":"To enable the k8s driver and pass args for k8s driver in buildx","Example":"","Deprecated":"false"},{"Env":"BUILDX_PROVENANCE_MODE","EnvType":"string","EnvValue":"","EnvDescription":"provinance is set to true by default by docker. this will add some build related data in generated build manifest.it also adds some unknown:unknown key:value pair which may not be compatible by some container registries. with buildx k8s driver , provinenance=true is causing issue when push manifest to quay registry, so setting it to false","Example":"","Deprecated":"false"},{"Env":"BUILD_LOG_TTL_VALUE_IN_SECS","EnvType":"int","EnvValue":"3600","EnvDescription":"This is the time that the pods of ci/pre-cd/post-cd live after completion state.","Example":"","Deprecated":"false"},{"Env":"CACHE_LIMIT","EnvType":"int64","EnvValue":"5000000000","EnvDescription":"Cache limit.","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for Pre/Post cd ","Example":"","Deprecated":"false"},{"Env":"CD_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Limit Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"Toleration key for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"Toleration value for Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"CPU Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"Memory Resource Rquest Pre/Post CD","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for Pre/Post CD(AWF,System)","Example":"","Deprecated":"false"},{"Env":"CD_WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"cd-runner","EnvDescription":"Service account to be used in Pre/Post CD pod","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_BASE_CIDR","EnvType":"string","EnvValue":"","EnvDescription":"To pass the IP cidr for CI","Example":"","Deprecated":"false"},{"Env":"CI_DEFAULT_ADDRESS_POOL_SIZE","EnvType":"int","EnvValue":"","EnvDescription":"The subnet size to allocate from the base pool for CI","Example":"","Deprecated":"false"},{"Env":"CI_IGNORE_DOCKER_CACHE","EnvType":"bool","EnvValue":"","EnvDescription":"Ignoring docker cache ","Example":"","Deprecated":"false"},{"Env":"CI_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for build logs","Example":"","Deprecated":"false"},{"Env":"CI_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"Node label selector for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"","EnvDescription":"Toleration key for CI","Example":"","Deprecated":"false"},{"Env":"CI_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"","EnvDescription":"Toleration value for CI","Example":"","Deprecated":"false"},{"Env":"CI_RUNNER_DOCKER_MTU_VALUE","EnvType":"int","EnvValue":"-1","EnvDescription":"this is to control the bytes of inofrmation passed in a network packet in ci-runner. default is -1 (defaults to the underlying node mtu value)","Example":"","Deprecated":"false"},{"Env":"CI_SUCCESS_AUTO_TRIGGER_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"this is to control the no of linked pipelines should be hanled in one go when a ci-success event of an parent ci is received","Example":"","Deprecated":"false"},{"Env":"CI_VOLUME_MOUNTS_JSON","EnvType":"string","EnvValue":"","EnvDescription":"additional volume mount data for CI and JOB","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_EXECUTOR_TYPE","EnvType":"","EnvValue":"AWF","EnvDescription":"Executor type for CI(AWF,System)","Example":"","Deprecated":"false"},{"Env":"DEFAULT_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"arsenal-v1/ci-artifacts","EnvDescription":"Key location for artifacts being created","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_BUCKET","EnvType":"string","EnvValue":"devtron-pro-ci-logs","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_BUILD_LOGS_KEY_PREFIX","EnvType":"string","EnvValue":"arsenal-v1","EnvDescription":"Bucket prefix for build logs","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET","EnvType":"string","EnvValue":"ci-caching","EnvDescription":"Bucket name for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CACHE_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"Build Cache bucket region","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_ARTIFACT_KEY_LOCATION","EnvType":"string","EnvValue":"","EnvDescription":"Bucket prefix for build cache","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_LOGS_BUCKET_REGION","EnvType":"string","EnvValue":"us-east-2","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_NAMESPACE","EnvType":"string","EnvValue":"","EnvDescription":"Namespace for devtron stack","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CD_TIMEOUT","EnvType":"int64","EnvValue":"3600","EnvDescription":"Timeout for Pre/Post-Cd to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_CI_IMAGE","EnvType":"string","EnvValue":"686244538589.dkr.ecr.us-east-2.amazonaws.com/cirunner:47","EnvDescription":"To pass the ci-runner image","Example":"","Deprecated":"false"},{"Env":"DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtron-ci","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TARGET_PLATFORM","EnvType":"string","EnvValue":"","EnvDescription":"Default architecture for buildx","Example":"","Deprecated":"false"},{"Env":"DOCKER_BUILD_CACHE_PATH","EnvType":"string","EnvValue":"/var/lib/docker","EnvDescription":"Path to store cache of docker build (/var/lib/docker-\u003e for legacy docker build, /var/lib/devtron-\u003e for buildx)","Example":"","Deprecated":"false"},{"Env":"ENABLE_BUILD_CONTEXT","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable build context in Devtron.","Example":"","Deprecated":"false"},{"Env":"ENABLE_WORKFLOW_EXECUTION_STAGE","EnvType":"bool","EnvValue":"true","EnvDescription":"if enabled then we will display build stages separately for CI/Job/Pre-Post CD","Example":"true","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_CM_NAME","EnvType":"string","EnvValue":"blob-storage-cm","EnvDescription":"name of the config map(contains bucket name, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_BLOB_STORAGE_SECRET_NAME","EnvType":"string","EnvValue":"blob-storage-secret","EnvDescription":"name of the secret(contains password, accessId,passKeys, etc.) in external cluster when there is some operation related to external cluster, for example:-downloading cd artifact pushed in external cluster's env and we need to download from there, downloads ci logs pushed in external cluster's blob","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_LABEL_SELECTOR","EnvType":"","EnvValue":"","EnvDescription":"This is an array of strings used when submitting a workflow for pre or post-CD execution. If the ","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_KEY","EnvType":"string","EnvValue":"dedicated","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CD_NODE_TAINTS_VALUE","EnvType":"string","EnvValue":"ci","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_API_SECRET","EnvType":"string","EnvValue":"devtroncd-secret","EnvDescription":"External CI API secret.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_PAYLOAD","EnvType":"string","EnvValue":"{\"ciProjectDetails\":[{\"gitRepository\":\"https://github.com/vikram1601/getting-started-nodejs.git\",\"checkoutPath\":\"./abc\",\"commitHash\":\"239077135f8cdeeccb7857e2851348f558cb53d3\",\"commitTime\":\"2022-10-30T20:00:00\",\"branch\":\"master\",\"message\":\"Update README.md\",\"author\":\"User Name \"}],\"dockerImage\":\"445808685819.dkr.ecr.us-east-2.amazonaws.com/orch:23907713-2\"}","EnvDescription":"External CI payload with project details.","Example":"","Deprecated":"false"},{"Env":"EXTERNAL_CI_WEB_HOOK_URL","EnvType":"string","EnvValue":"","EnvDescription":"default is {{HOST_URL}}/orchestrator/webhook/ext-ci. It is used for external ci.","Example":"","Deprecated":"false"},{"Env":"IGNORE_CM_CS_IN_CI_JOB","EnvType":"bool","EnvValue":"false","EnvDescription":"Ignore CM/CS in CI-pipeline as Job","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_COUNT","EnvType":"int","EnvValue":"0","EnvDescription":"push artifact(image) in ci retry count ","Example":"","Deprecated":"false"},{"Env":"IMAGE_RETRY_INTERVAL","EnvType":"int","EnvValue":"5","EnvDescription":"image retry interval takes value in seconds","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCANNER_ENDPOINT","EnvType":"string","EnvValue":"http://image-scanner-new-demo-devtroncd-service.devtroncd:80","EnvDescription":"Image-scanner micro-service URL","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_MAX_RETRIES","EnvType":"int","EnvValue":"3","EnvDescription":"Max retry count for image-scanning","Example":"","Deprecated":"false"},{"Env":"IMAGE_SCAN_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay for the image-scaning to start","Example":"","Deprecated":"false"},{"Env":"IN_APP_LOGGING_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"Used in case of argo workflow is enabled. If enabled logs push will be managed by us, else will be managed by argo workflow.","Example":"","Deprecated":"false"},{"Env":"MAX_CD_WORKFLOW_RUNNER_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time pre/post-cd-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MAX_CI_WORKFLOW_RETRIES","EnvType":"int","EnvValue":"0","EnvDescription":"Maximum time CI-workflow create pod if it fails to complete","Example":"","Deprecated":"false"},{"Env":"MODE","EnvType":"string","EnvValue":"DEV","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_SERVER_HOST","EnvType":"string","EnvValue":"localhost:4222","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ORCH_HOST","EnvType":"string","EnvValue":"http://devtroncd-orchestrator-service-prod.devtroncd/webhook/msg/nats","EnvDescription":"Orchestrator micro-service URL ","Example":"","Deprecated":"false"},{"Env":"ORCH_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"Orchestrator token","Example":"","Deprecated":"false"},{"Env":"PRE_CI_CACHE_PATH","EnvType":"string","EnvValue":"/devtroncd-cache","EnvDescription":"Cache path for Pre CI tasks","Example":"","Deprecated":"false"},{"Env":"SHOW_DOCKER_BUILD_ARGS","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable showing the args passed for CI in build logs","Example":"","Deprecated":"false"},{"Env":"SKIP_CI_JOB_BUILD_CACHE_PUSH_PULL","EnvType":"bool","EnvValue":"false","EnvDescription":"To skip cache Push/Pull for ci job","Example":"","Deprecated":"false"},{"Env":"SKIP_CREATING_ECR_REPO","EnvType":"bool","EnvValue":"false","EnvDescription":"By disabling this ECR repo won't get created if it's not available on ECR from build configuration","Example":"","Deprecated":"false"},{"Env":"TERMINATION_GRACE_PERIOD_SECS","EnvType":"int","EnvValue":"180","EnvDescription":"this is the time given to workflow pods to shutdown. (grace full termination time)","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_QUERY_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 query for listing artifacts","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CD_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post cd","Example":"","Deprecated":"false"},{"Env":"USE_BLOB_STORAGE_CONFIG_IN_CI_WORKFLOW","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable blob storage in pre and post ci","Example":"","Deprecated":"false"},{"Env":"USE_BUILDX","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable buildx feature globally","Example":"","Deprecated":"false"},{"Env":"USE_DOCKER_API_TO_GET_DIGEST","EnvType":"bool","EnvValue":"false","EnvDescription":"when user do not pass the digest then this flag controls , finding the image digest using docker API or not. if set to true we get the digest from docker API call else use docker pull command. [logic in ci-runner]","Example":"","Deprecated":"false"},{"Env":"USE_EXTERNAL_NODE","EnvType":"bool","EnvValue":"false","EnvDescription":"It is used in case of Pre/ Post Cd with run in application mode. If enabled the node lebels are read from EXTERNAL_CD_NODE_LABEL_SELECTOR else from CD_NODE_LABEL_SELECTOR MODE: if the vale is DEV, it will read the local kube config file or else from the cluser location.","Example":"","Deprecated":"false"},{"Env":"USE_IMAGE_TAG_FROM_GIT_PROVIDER_FOR_TAG_BASED_BUILD","EnvType":"bool","EnvValue":"false","EnvDescription":"To use the same tag in container image as that of git tag","Example":"","Deprecated":"false"},{"Env":"WF_CONTROLLER_INSTANCE_ID","EnvType":"string","EnvValue":"devtron-runner","EnvDescription":"Workflow controller instance ID.","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_CACHE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"flag is used to configure how Docker caches are handled during a CI/CD ","Example":"","Deprecated":"false"},{"Env":"WORKFLOW_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"ci-runner","EnvDescription":"","Example":"","Deprecated":"false"}]},{"Category":"DEVTRON","Fields":[{"Env":"-","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ADDITIONAL_NODE_GROUP_LABELS","EnvType":"","EnvValue":"","EnvDescription":"Add comma separated list of additional node group labels to default labels","Example":"karpenter.sh/nodepool,cloud.google.com/gke-nodepool","Deprecated":"false"},{"Env":"APP_SYNC_IMAGE","EnvType":"string","EnvValue":"quay.io/devtron/chart-sync:1227622d-132-3775","EnvDescription":"For the app sync image, this image will be used in app-manual sync job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_JOB_RESOURCES_OBJ","EnvType":"string","EnvValue":"","EnvDescription":"To pass the resource of app sync","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SERVICE_ACCOUNT","EnvType":"string","EnvValue":"chart-sync","EnvDescription":"Service account to be used in app sync Job","Example":"","Deprecated":"false"},{"Env":"APP_SYNC_SHUTDOWN_WAIT_DURATION","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"ARGO_AUTO_SYNC_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"If enabled all argocd application will have auto sync enabled","Example":"true","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_COUNT_ON_CONFLICT","EnvType":"int","EnvValue":"3","EnvDescription":"retry argocd app manual sync if the timeline is stuck in ARGOCD_SYNC_INITIATED state for more than this defined time (in mins)","Example":"","Deprecated":"false"},{"Env":"ARGO_GIT_COMMIT_RETRY_DELAY_ON_CONFLICT","EnvType":"int","EnvValue":"1","EnvDescription":"Delay on retrying the maifest commit the on gitops","Example":"","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_COUNT","EnvType":"int","EnvValue":"4","EnvDescription":"Retry count for registering a GitOps repository to ArgoCD","Example":"3","Deprecated":"false"},{"Env":"ARGO_REPO_REGISTER_RETRY_DELAY","EnvType":"int","EnvValue":"5","EnvDescription":"Delay (in Seconds) between the retries for registering a GitOps repository to ArgoCD","Example":"5","Deprecated":"false"},{"Env":"BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"there is feature to get URL's of services/ingresses. so to extract those, we need to parse all the servcie and ingress objects of the application. this BATCH_SIZE flag controls the no of these objects get parsed in one go.","Example":"","Deprecated":"false"},{"Env":"BLOB_STORAGE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host for the devtron stack","Example":"","Deprecated":"false"},{"Env":"CD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"CD_PORT","EnvType":"string","EnvValue":"8000","EnvDescription":"Port for pre/post-cd","Example":"","Deprecated":"false"},{"Env":"CExpirationTime","EnvType":"int","EnvValue":"600","EnvDescription":"Caching expiration time.","Example":"","Deprecated":"false"},{"Env":"CI_TRIGGER_CRON_TIME","EnvType":"int","EnvValue":"2","EnvDescription":"For image poll plugin","Example":"","Deprecated":"false"},{"Env":"CI_WORKFLOW_STATUS_UPDATE_CRON","EnvType":"string","EnvValue":"*/5 * * * *","EnvDescription":"Cron schedule for CI pipeline status","Example":"","Deprecated":"false"},{"Env":"CLI_CMD_TIMEOUT_GLOBAL_SECONDS","EnvType":"int","EnvValue":"0","EnvDescription":"Used in git cli opeartion timeout","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_BACKGROUND_REFRESH_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"Enable background refresh of cluster overview cache","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"Enable caching for cluster overview data","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_MAX_PARALLEL_CLUSTERS","EnvType":"int","EnvValue":"15","EnvDescription":"Maximum number of clusters to fetch in parallel during refresh","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_MAX_STALE_DATA_SECONDS","EnvType":"int","EnvValue":"30","EnvDescription":"Maximum age of cached data in seconds before warning","Example":"","Deprecated":"false"},{"Env":"CLUSTER_OVERVIEW_REFRESH_INTERVAL_SECONDS","EnvType":"int","EnvValue":"15","EnvDescription":"Background cache refresh interval in seconds","Example":"","Deprecated":"false"},{"Env":"CLUSTER_STATUS_CRON_TIME","EnvType":"int","EnvValue":"15","EnvDescription":"Cron schedule for cluster status on resource browser","Example":"","Deprecated":"false"},{"Env":"CONSUMER_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_LOG_TIME_LIMIT","EnvType":"int64","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEFAULT_TIMEOUT","EnvType":"float64","EnvValue":"3600","EnvDescription":"Timeout for CI to be completed","Example":"","Deprecated":"false"},{"Env":"DEVTRON_BOM_URL","EnvType":"string","EnvValue":"https://raw.githubusercontent.com/devtron-labs/devtron/%s/charts/devtron/devtron-bom.yaml","EnvDescription":"Path to devtron-bom.yaml of devtron charts, used for module installation and devtron upgrade","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_DEX_SECRET_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of dex secret","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_CHART_NAME","EnvType":"string","EnvValue":"devtron-operator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Name of the Devtron Helm release. ","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_RELEASE_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace of the Devtron Helm release","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_NAME","EnvType":"string","EnvValue":"devtron","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_HELM_REPO_URL","EnvType":"string","EnvValue":"https://helm.devtron.ai","EnvDescription":"Is used to install modules (stack manager)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLATION_TYPE","EnvType":"string","EnvValue":"","EnvDescription":"Devtron Installation type(EA/Full)","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLER_MODULES_PATH","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"Path to devtron installer modules, used to find the helm charts and values files","Example":"","Deprecated":"false"},{"Env":"DEVTRON_INSTALLER_RELEASE_PATH","EnvType":"string","EnvValue":"installer.release","EnvDescription":"Path to devtron installer release, used to find the helm charts and values files","Example":"","Deprecated":"false"},{"Env":"DEVTRON_MODULES_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.modules","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_OPERATOR_BASE_PATH","EnvType":"string","EnvValue":"","EnvDescription":"Base path for devtron operator, used to find the helm charts and values files","Example":"","Deprecated":"false"},{"Env":"DEVTRON_SECRET_NAME","EnvType":"string","EnvValue":"devtron-secret","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEVTRON_VERSION_IDENTIFIER_IN_HELM_VALUES","EnvType":"string","EnvValue":"installer.release","EnvDescription":"devtron operator version identifier in helm values yaml","Example":"","Deprecated":"false"},{"Env":"DEX_CID","EnvType":"string","EnvValue":"example-app","EnvDescription":"dex client id ","Example":"","Deprecated":"false"},{"Env":"DEX_CLIENT_ID","EnvType":"string","EnvValue":"argo-cd","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_CSTOREKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX CSTOREKEY.","Example":"","Deprecated":"false"},{"Env":"DEX_JWTKEY","EnvType":"string","EnvValue":"","EnvDescription":"DEX JWT key. ","Example":"","Deprecated":"false"},{"Env":"DEX_RURL","EnvType":"string","EnvValue":"http://127.0.0.1:8080/callback","EnvDescription":"Dex redirect URL(http://argocd-dex-server.devtroncd:8080/callback)","Example":"","Deprecated":"false"},{"Env":"DEX_SCOPES","EnvType":"","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_SECRET","EnvType":"string","EnvValue":"","EnvDescription":"Dex secret","Example":"","Deprecated":"false"},{"Env":"DEX_URL","EnvType":"string","EnvValue":"","EnvDescription":"Dex service endpoint with dex path(http://argocd-dex-server.devtroncd:5556/dex)","Example":"","Deprecated":"false"},{"Env":"ECR_REPO_NAME_PREFIX","EnvType":"string","EnvValue":"test/","EnvDescription":"Prefix for ECR repo to be created in does not exist","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_ARGO_CD_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_ASYNC_INSTALL_DEVTRON_CHART","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable async installation of no-gitops application","Example":"","Deprecated":"false"},{"Env":"ENABLE_LINKED_CI_ARTIFACT_COPY","EnvType":"bool","EnvValue":"false","EnvDescription":"Enable copying artifacts from parent CI pipeline to linked CI pipeline during creation","Example":"","Deprecated":"false"},{"Env":"ENABLE_PASSWORD_ENCRYPTION","EnvType":"bool","EnvValue":"true","EnvDescription":"enable password encryption","Example":"","Deprecated":"false"},{"Env":"EPHEMERAL_SERVER_VERSION_REGEX","EnvType":"string","EnvValue":"v[1-9]\\.\\b(2[3-9]\\|[3-9][0-9])\\b.*","EnvDescription":"ephemeral containers support version regex that is compared with k8sServerVersion","Example":"","Deprecated":"false"},{"Env":"EVENT_URL","EnvType":"string","EnvValue":"http://localhost:3000/notify","EnvDescription":"Notifier service url","Example":"","Deprecated":"false"},{"Env":"EXECUTE_WIRE_NIL_CHECKER","EnvType":"bool","EnvValue":"false","EnvDescription":"checks for any nil pointer in wire.go","Example":"","Deprecated":"false"},{"Env":"EXPOSE_CI_METRICS","EnvType":"bool","EnvValue":"false","EnvDescription":"To expose CI metrics","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"restart workload retrieval batch size ","Example":"","Deprecated":"false"},{"Env":"FEATURE_RESTART_WORKLOAD_WORKER_POOL_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"restart workload retrieval pool size","Example":"","Deprecated":"false"},{"Env":"FORCE_SECURITY_SCANNING","EnvType":"bool","EnvValue":"false","EnvDescription":"By enabling this no one can disable image scaning on ci-pipeline from UI","Example":"","Deprecated":"false"},{"Env":"GITHUB_ORG_NAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITHUB_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITHUB_USERNAME","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GITOPS_REPO_PREFIX","EnvType":"string","EnvValue":"","EnvDescription":"Prefix for Gitops repo being creation for argocd application","Example":"","Deprecated":"false"},{"Env":"GO_RUNTIME_ENV","EnvType":"string","EnvValue":"production","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GRAFANA_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Namespace for grafana","Example":"","Deprecated":"false"},{"Env":"GRAFANA_ORG_ID","EnvType":"int","EnvValue":"2","EnvDescription":"Org ID for grafana for application metrics","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PASSWORD","EnvType":"string","EnvValue":"prom-operator","EnvDescription":"Password for grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_PORT","EnvType":"string","EnvValue":"8090","EnvDescription":"Port for grafana micro-service","Example":"","Deprecated":"false"},{"Env":"GRAFANA_URL","EnvType":"string","EnvValue":"","EnvDescription":"Host URL for the grafana dashboard","Example":"","Deprecated":"false"},{"Env":"GRAFANA_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"Username for grafana ","Example":"","Deprecated":"false"},{"Env":"HIDE_API_TOKENS","EnvType":"bool","EnvValue":"false","EnvDescription":"Boolean flag for should the api tokens generated be hidden from the UI","Example":"","Deprecated":"false"},{"Env":"HIDE_IMAGE_TAGGING_HARD_DELETE","EnvType":"bool","EnvValue":"false","EnvDescription":"Flag to hide the hard delete option in the image tagging service","Example":"","Deprecated":"false"},{"Env":"IGNORE_AUTOCOMPLETE_AUTH_CHECK","EnvType":"bool","EnvValue":"false","EnvDescription":"flag for ignoring auth check in autocomplete apis.","Example":"","Deprecated":"false"},{"Env":"INSTALLED_MODULES","EnvType":"","EnvValue":"","EnvDescription":"List of installed modules given in helm values/yaml are written in cm and used by devtron to know which modules are given","Example":"security.trivy,security.clair","Deprecated":"false"},{"Env":"INSTALLER_CRD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"namespace where Custom Resource Definitions get installed","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_GROUP_NAME","EnvType":"string","EnvValue":"installer.devtron.ai","EnvDescription":"Devtron installer CRD group name, partially deprecated.","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_RESOURCE","EnvType":"string","EnvValue":"installers","EnvDescription":"Devtron installer CRD resource name, partially deprecated","Example":"","Deprecated":"false"},{"Env":"INSTALLER_CRD_OBJECT_VERSION","EnvType":"string","EnvValue":"v1alpha1","EnvDescription":"version of the CRDs. default is v1alpha1","Example":"","Deprecated":"false"},{"Env":"IS_AIR_GAP_ENVIRONMENT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"JwtExpirationTime","EnvType":"int","EnvValue":"120","EnvDescription":"JWT expiration time.","Example":"","Deprecated":"false"},{"Env":"K8s_CLIENT_MAX_IDLE_CONNS_PER_HOST","EnvType":"int","EnvValue":"25","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_IDLE_CONN_TIMEOUT","EnvType":"int","EnvValue":"300","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_KEEPALIVE","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TCP_TIMEOUT","EnvType":"int","EnvValue":"30","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"K8s_TLS_HANDSHAKE_TIMEOUT","EnvType":"int","EnvValue":"10","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LENS_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Lens microservice timeout.","Example":"","Deprecated":"false"},{"Env":"LENS_URL","EnvType":"string","EnvValue":"http://lens-milandevtron-service:80","EnvDescription":"Lens micro-service URL","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LIMIT_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"LINKED_CI_ARTIFACT_COPY_LIMIT","EnvType":"int","EnvValue":"10","EnvDescription":"Maximum number of artifacts to copy from parent CI pipeline to linked CI pipeline","Example":"","Deprecated":"false"},{"Env":"LOGGER_DEV_MODE","EnvType":"bool","EnvValue":"false","EnvDescription":"Enables a different logger theme.","Example":"","Deprecated":"false"},{"Env":"LOG_LEVEL","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"MAX_SESSION_PER_USER","EnvType":"int","EnvValue":"5","EnvDescription":"max no of cluster terminal pods can be created by an user","Example":"","Deprecated":"false"},{"Env":"MODULE_METADATA_API_URL","EnvType":"string","EnvValue":"https://api.devtron.ai/module?name=%s","EnvDescription":"Modules list and meta info will be fetched from this server, that is central api server of devtron.","Example":"","Deprecated":"false"},{"Env":"MODULE_STATUS_HANDLING_CRON_DURATION_MIN","EnvType":"int","EnvValue":"3","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_ACK_WAIT_IN_SECS","EnvType":"int","EnvValue":"120","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_BUFFER_SIZE","EnvType":"int","EnvValue":"-1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_MAX_AGE","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_PROCESSING_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NATS_MSG_REPLICAS","EnvType":"int","EnvValue":"0","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"NOTIFICATION_MEDIUM","EnvType":"NotificationMedium","EnvValue":"rest","EnvDescription":"notification medium","Example":"","Deprecated":"false"},{"Env":"OTEL_COLLECTOR_URL","EnvType":"string","EnvValue":"","EnvDescription":"Opentelemetry URL ","Example":"","Deprecated":"false"},{"Env":"PARALLELISM_LIMIT_FOR_TAG_PROCESSING","EnvType":"int","EnvValue":"","EnvDescription":"App manual sync job parallel tag processing count.","Example":"","Deprecated":"false"},{"Env":"PG_EXPORT_PROM_METRICS","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_FAILURE_QUERIES","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_ALL_QUERY","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_LOG_SLOW_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PG_QUERY_DUR_THRESHOLD","EnvType":"int64","EnvValue":"5000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"PLUGIN_NAME","EnvType":"string","EnvValue":"Pull images from container repository","EnvDescription":"Handles image retrieval from a container repository and triggers subsequent CI processes upon detecting new images.Current default plugin name: Pull Images from Container Repository.","Example":"","Deprecated":"false"},{"Env":"PROPAGATE_EXTRA_LABELS","EnvType":"bool","EnvValue":"false","EnvDescription":"Add additional propagate labels like api.devtron.ai/appName, api.devtron.ai/envName, api.devtron.ai/project along with the user defined ones.","Example":"","Deprecated":"false"},{"Env":"PROXY_SERVICE_CONFIG","EnvType":"string","EnvValue":"{}","EnvDescription":"Proxy configuration for micro-service to be accessible on orhcestrator ingress","Example":"","Deprecated":"false"},{"Env":"REQ_CI_CPU","EnvType":"string","EnvValue":"0.5","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"REQ_CI_MEM","EnvType":"string","EnvValue":"3G","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"RESTRICT_TERMINAL_ACCESS_FOR_NON_SUPER_USER","EnvType":"bool","EnvValue":"false","EnvDescription":"To restrict the cluster terminal from user having non-super admin acceess","Example":"","Deprecated":"false"},{"Env":"RUNTIME_CONFIG_LOCAL_DEV","EnvType":"LocalDevMode","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_ENABLED","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable scoped variable option","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_FORMAT","EnvType":"string","EnvValue":"@{{%s}}","EnvDescription":"Its a scope format for varialbe name.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_HANDLE_PRIMITIVES","EnvType":"bool","EnvValue":"false","EnvDescription":"This describe should we handle primitives or not in scoped variable template parsing.","Example":"","Deprecated":"false"},{"Env":"SCOPED_VARIABLE_NAME_REGEX","EnvType":"string","EnvValue":"^[a-zA-Z][a-zA-Z0-9_-]{0,62}[a-zA-Z0-9]$","EnvDescription":"Regex for scoped variable name that must passed this regex.","Example":"","Deprecated":"false"},{"Env":"SOCKET_DISCONNECT_DELAY_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"The server closes a session when a client receiving connection have not been seen for a while.This delay is configured by this setting. By default the session is closed when a receiving connection wasn't seen for 5 seconds.","Example":"","Deprecated":"false"},{"Env":"SOCKET_HEARTBEAT_SECONDS","EnvType":"int","EnvValue":"25","EnvDescription":"In order to keep proxies and load balancers from closing long running http requests we need to pretend that the connection is active and send a heartbeat packet once in a while. This setting controls how often this is done. By default a heartbeat packet is sent every 25 seconds.","Example":"","Deprecated":"false"},{"Env":"STREAM_CONFIG_JSON","EnvType":"string","EnvValue":"","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"SYSTEM_VAR_PREFIX","EnvType":"string","EnvValue":"DEVTRON_","EnvDescription":"Scoped variable prefix, variable name must have this prefix.","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_DEFAULT_NAMESPACE","EnvType":"string","EnvValue":"default","EnvDescription":"Cluster terminal default namespace","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_INACTIVE_DURATION_IN_MINS","EnvType":"int","EnvValue":"10","EnvDescription":"Timeout for cluster terminal to be inactive","Example":"","Deprecated":"false"},{"Env":"TERMINAL_POD_STATUS_SYNC_In_SECS","EnvType":"int","EnvValue":"600","EnvDescription":"this is the time interval at which the status of the cluster terminal pod","Example":"","Deprecated":"false"},{"Env":"TEST_APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_LOG_QUERY","EnvType":"bool","EnvValue":"true","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PASSWORD","EnvType":"string","EnvValue":"postgrespw","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_PORT","EnvType":"string","EnvValue":"55000","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TEST_PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_FOR_FAILED_CI_BUILD","EnvType":"string","EnvValue":"15","EnvDescription":"Timeout for Failed CI build ","Example":"","Deprecated":"false"},{"Env":"TIMEOUT_IN_SECONDS","EnvType":"int","EnvValue":"5","EnvDescription":"timeout to compute the urls from services and ingress objects of an application","Example":"","Deprecated":"false"},{"Env":"USER_SESSION_DURATION_SECONDS","EnvType":"int","EnvValue":"86400","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_ARTIFACT_LISTING_API_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 API for listing artifacts in Listing the images in pipeline","Example":"","Deprecated":"false"},{"Env":"USE_CUSTOM_HTTP_TRANSPORT","EnvType":"bool","EnvValue":"false","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"USE_GIT_CLI","EnvType":"bool","EnvValue":"false","EnvDescription":"To enable git cli","Example":"","Deprecated":"false"},{"Env":"USE_RBAC_CREATION_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To use the V2 for RBAC creation","Example":"","Deprecated":"false"},{"Env":"VARIABLE_CACHE_ENABLED","EnvType":"bool","EnvValue":"true","EnvDescription":"This is used to control caching of all the scope variables defined in the system.","Example":"","Deprecated":"false"},{"Env":"VARIABLE_EXPRESSION_REGEX","EnvType":"string","EnvValue":"@{{([^}]+)}}","EnvDescription":"Scoped variable expression regex","Example":"","Deprecated":"false"},{"Env":"WEBHOOK_TOKEN","EnvType":"string","EnvValue":"","EnvDescription":"If you want to continue using jenkins for CI then please provide this for authentication of requests","Example":"","Deprecated":"false"}]},{"Category":"GITOPS","Fields":[{"Env":"ACD_CM","EnvType":"string","EnvValue":"argocd-cm","EnvDescription":"Name of the argocd CM","Example":"","Deprecated":"false"},{"Env":"ACD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"To pass the argocd namespace","Example":"","Deprecated":"false"},{"Env":"ACD_PASSWORD","EnvType":"string","EnvValue":"","EnvDescription":"Password for the Argocd (deprecated)","Example":"","Deprecated":"false"},{"Env":"ACD_USERNAME","EnvType":"string","EnvValue":"admin","EnvDescription":"User name for argocd","Example":"","Deprecated":"false"},{"Env":"GITOPS_SECRET_NAME","EnvType":"string","EnvValue":"devtron-gitops-secret","EnvDescription":"devtron-gitops-secret","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS","EnvType":"string","EnvValue":"Deployment,Rollout,StatefulSet,ReplicaSet","EnvDescription":"this holds the list of k8s resource names which support replicas key. this list used in hibernate/un hibernate process","Example":"","Deprecated":"false"},{"Env":"RESOURCE_LIST_FOR_REPLICAS_BATCH_SIZE","EnvType":"int","EnvValue":"5","EnvDescription":"this the batch size to control no of above resources can be parsed in one go to determine hibernate status","Example":"","Deprecated":"false"}]},{"Category":"INFRA_SETUP","Fields":[{"Env":"DASHBOARD_HOST","EnvType":"string","EnvValue":"localhost","EnvDescription":"Dashboard micro-service URL","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_NAMESPACE","EnvType":"string","EnvValue":"devtroncd","EnvDescription":"Dashboard micro-service namespace","Example":"","Deprecated":"false"},{"Env":"DASHBOARD_PORT","EnvType":"string","EnvValue":"3000","EnvDescription":"Port for dashboard micro-service","Example":"","Deprecated":"false"},{"Env":"DEX_HOST","EnvType":"string","EnvValue":"http://localhost","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"DEX_PORT","EnvType":"string","EnvValue":"5556","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_PROTOCOL","EnvType":"string","EnvValue":"REST","EnvDescription":"Protocol to connect with git-sensor micro-service","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_SERVICE_CONFIG","EnvType":"string","EnvValue":"{\"loadBalancingPolicy\":\"pick_first\"}","EnvDescription":"git-sensor grpc service config","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_TIMEOUT","EnvType":"int","EnvValue":"0","EnvDescription":"Timeout for getting response from the git-sensor","Example":"","Deprecated":"false"},{"Env":"GIT_SENSOR_URL","EnvType":"string","EnvValue":"127.0.0.1:7070","EnvDescription":"git-sensor micro-service url ","Example":"","Deprecated":"false"},{"Env":"HELM_CLIENT_URL","EnvType":"string","EnvValue":"127.0.0.1:50051","EnvDescription":"Kubelink micro-service url ","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_RECEIVE_MSG_SIZE","EnvType":"int","EnvValue":"20","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_MAX_SEND_MSG_SIZE","EnvType":"int","EnvValue":"4","EnvDescription":"","Example":"","Deprecated":"false"},{"Env":"KUBELINK_GRPC_SERVICE_CONFIG","EnvType":"string","EnvValue":"{\"loadBalancingPolicy\":\"round_robin\"}","EnvDescription":"kubelink grpc service config","Example":"","Deprecated":"false"}]},{"Category":"POSTGRES","Fields":[{"Env":"APP","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"Application name","Example":"","Deprecated":"false"},{"Env":"CASBIN_DATABASE","EnvType":"string","EnvValue":"casbin","EnvDescription":"Database for casbin","Example":"","Deprecated":"false"},{"Env":"PG_ADDR","EnvType":"string","EnvValue":"127.0.0.1","EnvDescription":"address of postgres service","Example":"postgresql-postgresql.devtroncd","Deprecated":"false"},{"Env":"PG_DATABASE","EnvType":"string","EnvValue":"orchestrator","EnvDescription":"postgres database to be made connection with","Example":"orchestrator, casbin, git_sensor, lens","Deprecated":"false"},{"Env":"PG_PASSWORD","EnvType":"string","EnvValue":"{password}","EnvDescription":"password for postgres, associated with PG_USER","Example":"confidential ;)","Deprecated":"false"},{"Env":"PG_PORT","EnvType":"string","EnvValue":"5432","EnvDescription":"port of postgresql service","Example":"5432","Deprecated":"false"},{"Env":"PG_READ_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for read operation in postgres","Example":"","Deprecated":"false"},{"Env":"PG_SSL_MODE","EnvType":"string","EnvValue":"","EnvDescription":"ssl mode for postgres connection","Example":"disable, require, verify-ca, verify-full","Deprecated":"false"},{"Env":"PG_SSL_ROOT_CERT","EnvType":"string","EnvValue":"","EnvDescription":"path to the PEM CA bundle, required for verify-ca/verify-full ssl modes (for AWS RDS use the downloaded global-bundle.pem)","Example":"/etc/devtron/certs/rds-ca-bundle.pem","Deprecated":"false"},{"Env":"PG_USER","EnvType":"string","EnvValue":"postgres","EnvDescription":"user for postgres","Example":"postgres","Deprecated":"false"},{"Env":"PG_WRITE_TIMEOUT","EnvType":"int64","EnvValue":"30","EnvDescription":"Time out for write operation in postgres","Example":"","Deprecated":"false"}]},{"Category":"RBAC","Fields":[{"Env":"ENFORCER_CACHE","EnvType":"bool","EnvValue":"false","EnvDescription":"To Enable enforcer cache.","Example":"","Deprecated":"false"},{"Env":"ENFORCER_CACHE_EXPIRATION_IN_SEC","EnvType":"int","EnvValue":"86400","EnvDescription":"Expiration time (in seconds) for enforcer cache. ","Example":"","Deprecated":"false"},{"Env":"ENFORCER_MAX_BATCH_SIZE","EnvType":"int","EnvValue":"1","EnvDescription":"Maximum batch size for the enforcer.","Example":"","Deprecated":"false"},{"Env":"USE_CASBIN_V2","EnvType":"bool","EnvValue":"true","EnvDescription":"To enable casbin V2 API","Example":"","Deprecated":"false"}]}] \ No newline at end of file diff --git a/env_gen.md b/env_gen.md index 28debeb117..17871a512e 100644 --- a/env_gen.md +++ b/env_gen.md @@ -15,6 +15,7 @@ | FEATURE_MIGRATE_ARGOCD_APPLICATION_ENABLE | bool |false | enable migration of external argocd application to devtron pipeline | | false | | FEATURE_MIGRATE_FLUX_APPLICATION_ENABLE | bool |false | enable flux application services | | false | | FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | eligible time for checking flux app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. FLUX_CD_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle. | | false | + | FOREGROUND_DELETE_CD_PIPELINE | bool |false | If enabled, deleting a CD pipeline will not delete the underlying deployed resource (helm release/argocd application/flux release) unless the request explicitly overrides it via payload. | | false | | HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME | string |120 | eligible time for checking helm app status periodically and update in db, value is in seconds., default is 120, if wfr is updated within configured time i.e. HELM_PIPELINE_STATUS_CHECK_ELIGIBLE_TIME then do not include for this cron cycle. | | false | | IS_INTERNAL_USE | bool |true | If enabled then cd pipeline and helm apps will not need the deployment app type mandatorily. Couple this flag with HIDE_GITOPS_OR_HELM_OPTION (in Dashborad) and if gitops is configured and allowed for the env, pipeline/ helm app will gitops else no-gitops. | | false | | MIGRATE_DEPLOYMENT_CONFIG_DATA | bool |false | migrate deployment config data from charts table to deployment_config table | | false | diff --git a/pkg/bean/app.go b/pkg/bean/app.go index 82b6c1e0fc..e5984f8731 100644 --- a/pkg/bean/app.go +++ b/pkg/bean/app.go @@ -770,6 +770,16 @@ type CDPatchRequest struct { UserId int32 `json:"-"` ForceDelete bool `json:"-"` NonCascadeDelete bool `json:"-"` + ForegroundDelete *bool `json:"-"` +} + +// GetForegroundDelete resolves the effective foreground-delete flag for this +// request: the explicit query-param value if set, otherwise the provided env default. +func (r *CDPatchRequest) GetForegroundDelete(envDefault bool) bool { + if r.ForegroundDelete != nil { + return *r.ForegroundDelete + } + return envDefault } type CdPatchAction int diff --git a/pkg/bean/app_test.go b/pkg/bean/app_test.go new file mode 100644 index 0000000000..0d357def24 --- /dev/null +++ b/pkg/bean/app_test.go @@ -0,0 +1,28 @@ +package bean + +import "testing" + +func boolPtr(b bool) *bool { return &b } + +func TestCDPatchRequest_GetForegroundDelete(t *testing.T) { + tests := []struct { + name string + field *bool + envDefault bool + want bool + }{ + {"nil payload, env false", nil, false, false}, + {"nil payload, env true", nil, true, true}, + {"payload true overrides env false", boolPtr(true), false, true}, + {"payload false overrides env true", boolPtr(false), true, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &CDPatchRequest{ForegroundDelete: tt.field} + if got := r.GetForegroundDelete(tt.envDefault); got != tt.want { + t.Errorf("GetForegroundDelete(%v) with field=%v = %v, want %v", + tt.envDefault, tt.field, got, tt.want) + } + }) + } +} diff --git a/pkg/bulkAction/service/BulkUpdateService.go b/pkg/bulkAction/service/BulkUpdateService.go index 817ebfb1cb..ccb50a2e09 100644 --- a/pkg/bulkAction/service/BulkUpdateService.go +++ b/pkg/bulkAction/service/BulkUpdateService.go @@ -109,6 +109,7 @@ type BulkUpdateServiceImpl struct { deployedAppService deployedApp.DeployedAppService cdPipelineEventPublishService out.CDPipelineEventPublishService ciHandlerService trigger.HandlerService + envVariables *util2.EnvironmentVariables *BulkUpdateServiceEntImpl } @@ -131,6 +132,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository repository.BulkEditRepository deployedAppService deployedApp.DeployedAppService, cdPipelineEventPublishService out.CDPipelineEventPublishService, ciHandlerService trigger.HandlerService, + envVariables *util2.EnvironmentVariables, bulkUpdateServiceEntImpl *BulkUpdateServiceEntImpl, ) *BulkUpdateServiceImpl { return &BulkUpdateServiceImpl{ @@ -153,6 +155,7 @@ func NewBulkUpdateServiceImpl(bulkUpdateRepository repository.BulkEditRepository deployedAppService: deployedAppService, cdPipelineEventPublishService: cdPipelineEventPublishService, ciHandlerService: ciHandlerService, + envVariables: envVariables, BulkUpdateServiceEntImpl: bulkUpdateServiceEntImpl, } } @@ -1750,7 +1753,7 @@ func (impl BulkUpdateServiceImpl) PerformBulkDeleteActionOnCdPipelines(impactedP } if !dryRun { // Delete Cd pipeline - deleteResponse, err := impl.pipelineBuilder.DeleteCdPipeline(pipeline, ctx, deleteAction, true, userId) + deleteResponse, err := impl.pipelineBuilder.DeleteCdPipeline(pipeline, ctx, deleteAction, true, impl.envVariables.DeploymentServiceTypeConfig.ForegroundDeleteCdPipeline, userId) if err != nil { impl.logger.Errorw("error in deleting cd pipeline", "err", err, "pipelineId", pipeline.Id) respDto.DeletionResult = fmt.Sprintf("Not able to delete pipeline, %v", err) diff --git a/pkg/commonService/CommonBaseService.go b/pkg/commonService/CommonBaseService.go index 5e2847922b..267b86f12c 100644 --- a/pkg/commonService/CommonBaseService.go +++ b/pkg/commonService/CommonBaseService.go @@ -80,6 +80,7 @@ func (impl *CommonBaseServiceImpl) EnvironmentVariableList() (*EnvironmentVariab } environmentVariableList.FeatureGitOpsFlags = featureGitOpsFlags environmentVariableList.IsAirGapEnvironment = impl.globalEnvVariables.GlobalEnvVariables.IsAirGapEnvironment + environmentVariableList.ForegroundDeleteCdPipeline = impl.globalEnvVariables.DeploymentServiceTypeConfig.ForegroundDeleteCdPipeline return environmentVariableList, nil } diff --git a/pkg/commonService/bean_ent.go b/pkg/commonService/bean_ent.go index ec78ebb4ae..238d0966b1 100644 --- a/pkg/commonService/bean_ent.go +++ b/pkg/commonService/bean_ent.go @@ -17,5 +17,6 @@ package commonService type EnvironmentVariableListEnt struct { - IsAirGapEnvironment bool `json:"isAirGapEnvironment"` + IsAirGapEnvironment bool `json:"isAirGapEnvironment"` + ForegroundDeleteCdPipeline bool `json:"foregroundDeleteCdPipeline"` } diff --git a/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go b/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go index 849c0ac1ab..2958393983 100644 --- a/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go +++ b/pkg/eventProcessor/in/DeployedApplicationEventProcessorService.go @@ -281,7 +281,7 @@ func (impl *DeployedApplicationEventProcessorImpl) updateDevtronAppArgoAppDelete impl.logger.Infow("Deployment delete not requested for app, not deleting app from DB", "appName", application.Name, "app", application) return nil } - _, err = impl.pipelineBuilder.DeleteCdPipeline(&pipelineModel, context.Background(), bean.FORCE_DELETE, false, 1) + _, err = impl.pipelineBuilder.DeleteCdPipeline(&pipelineModel, context.Background(), bean.FORCE_DELETE, false, false, 1) if err != nil { impl.logger.Errorw("error in deleting cd pipeline", "err", err) return err diff --git a/pkg/eventProcessor/in/WorkflowEventProcessorService.go b/pkg/eventProcessor/in/WorkflowEventProcessorService.go index 67f533ef2c..94eb7c3e43 100644 --- a/pkg/eventProcessor/in/WorkflowEventProcessorService.go +++ b/pkg/eventProcessor/in/WorkflowEventProcessorService.go @@ -894,11 +894,16 @@ func (impl *WorkflowEventProcessorImpl) SubscribeCDPipelineDeleteEvent() error { } if util3.IsHelmApp(deploymentAppType) || util3.IsAcdApp(deploymentAppType) { impl.RemoveReleaseContextForPipeline(cdPipelineDeleteEvent.PipelineId, cdPipelineDeleteEvent.TriggeredBy) - // there is a possibility that when the pipeline was deleted, async request nats message was not consumed completely and could have led to dangling deployment app - // trying to delete deployment app once - err = impl.cdPipelineConfigService.DeleteHelmTypePipelineDeploymentApp(context.Background(), true, pipeline) - if err != nil { - impl.logger.Errorw("error, DeleteHelmTypePipelineDeploymentApp", "pipelineId", pipeline.Id) + if cdPipelineDeleteEvent.ForegroundDelete { + impl.logger.Infow("foreground delete: skipping deployment app deletion in async delete event, detaching pipeline", + "pipelineId", pipeline.Id, "deploymentAppName", pipeline.DeploymentAppName) + } else { + // there is a possibility that when the pipeline was deleted, async request nats message was not consumed completely and could have led to dangling deployment app + // trying to delete deployment app once + err = impl.cdPipelineConfigService.DeleteHelmTypePipelineDeploymentApp(context.Background(), true, pipeline) + if err != nil { + impl.logger.Errorw("error, DeleteHelmTypePipelineDeploymentApp", "pipelineId", pipeline.Id) + } } } } diff --git a/pkg/eventProcessor/out/PipelineConfigEventPublishService.go b/pkg/eventProcessor/out/PipelineConfigEventPublishService.go index 1106309732..15357e97c9 100644 --- a/pkg/eventProcessor/out/PipelineConfigEventPublishService.go +++ b/pkg/eventProcessor/out/PipelineConfigEventPublishService.go @@ -24,7 +24,7 @@ import ( ) type PipelineConfigEventPublishService interface { - PublishCDPipelineDelete(pipelineId int, triggeredBy int32) error + PublishCDPipelineDelete(pipelineId int, triggeredBy int32, foregroundDelete bool) error } type PipelineConfigEventPublishServiceImpl struct { @@ -41,11 +41,12 @@ func NewPipelineConfigEventPublishServiceImpl(logger *zap.SugaredLogger, } -func (impl *PipelineConfigEventPublishServiceImpl) PublishCDPipelineDelete(pipelineId int, triggeredBy int32) error { +func (impl *PipelineConfigEventPublishServiceImpl) PublishCDPipelineDelete(pipelineId int, triggeredBy int32, foregroundDelete bool) error { impl.logger.Infow("cd pipeline delete event handle", "pipelineId", pipelineId, "triggeredBy", triggeredBy) req := &bean.CdPipelineDeleteEvent{ - PipelineId: pipelineId, - TriggeredBy: triggeredBy, + PipelineId: pipelineId, + TriggeredBy: triggeredBy, + ForegroundDelete: foregroundDelete, } data, err := json.Marshal(req) if err != nil { diff --git a/pkg/eventProcessor/out/bean/bean.go b/pkg/eventProcessor/out/bean/bean.go index b2606f6f2e..b99dc20ff0 100644 --- a/pkg/eventProcessor/out/bean/bean.go +++ b/pkg/eventProcessor/out/bean/bean.go @@ -44,8 +44,9 @@ type DeploymentGroupAppWithEnv struct { } type CdPipelineDeleteEvent struct { - PipelineId int `json:"pipelineId"` - TriggeredBy int32 `json:"triggeredBy"` + PipelineId int `json:"pipelineId"` + TriggeredBy int32 `json:"triggeredBy"` + ForegroundDelete bool `json:"foregroundDelete"` } type CIPipelineGitWebhookEvent struct { diff --git a/pkg/pipeline/DeploymentPipelineConfigService.go b/pkg/pipeline/DeploymentPipelineConfigService.go index 45d81d01ff..2077e12686 100644 --- a/pkg/pipeline/DeploymentPipelineConfigService.go +++ b/pkg/pipeline/DeploymentPipelineConfigService.go @@ -122,7 +122,7 @@ type CdPipelineConfigService interface { // PatchCdPipelines : Handle CD pipeline patch requests, making necessary changes to the configuration and returning the updated version. // Performs Create ,Update and Delete operation. PatchCdPipelines(cdPipelines *bean.CDPatchRequest, ctx context.Context) (*bean.CdPipelines, error) - DeleteCdPipeline(pipeline *pipelineConfig.Pipeline, ctx context.Context, deleteAction int, acdDelete bool, userId int32) (*bean.AppDeleteResponseDTO, error) + DeleteCdPipeline(pipeline *pipelineConfig.Pipeline, ctx context.Context, deleteAction int, acdDelete bool, foregroundDelete bool, userId int32) (*bean.AppDeleteResponseDTO, error) DeleteACDAppCdPipelineWithNonCascade(pipeline *pipelineConfig.Pipeline, ctx context.Context, forceDelete bool, userId int32) (err error) // GetTriggerViewCdPipelinesForApp : GetTriggerViewCdPipelinesForApp(appId int) (cdPipelines *bean.CdPipelines, err error) @@ -1483,6 +1483,7 @@ func (impl *CdPipelineConfigServiceImpl) PatchCdPipelines(cdPipelines *bean.CDPa } else if cdPipelines.NonCascadeDelete { deleteAction = bean.NON_CASCADE_DELETE } + foregroundDelete := cdPipelines.GetForegroundDelete(impl.deploymentConfig.ForegroundDeleteCdPipeline) switch cdPipelines.Action { case bean.CD_CREATE: return impl.CreateCdPipelines(pipelineRequest, ctx) @@ -1495,7 +1496,7 @@ func (impl *CdPipelineConfigServiceImpl) PatchCdPipelines(cdPipelines *bean.CDPa impl.logger.Errorw("error in getting cd pipeline by id", "err", err, "id", cdPipelines.Pipeline.Id) return pipelineRequest, err } - deleteResponse, err := impl.DeleteCdPipeline(pipeline, ctx, deleteAction, false, cdPipelines.UserId) + deleteResponse, err := impl.DeleteCdPipeline(pipeline, ctx, deleteAction, false, foregroundDelete, cdPipelines.UserId) pipelineRequest.AppDeleteResponse = deleteResponse return pipelineRequest, err case bean.CD_DELETE_PARTIAL: @@ -1512,7 +1513,7 @@ func (impl *CdPipelineConfigServiceImpl) PatchCdPipelines(cdPipelines *bean.CDPa } } -func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConfig.Pipeline, ctx context.Context, deleteAction int, deleteFromAcd bool, userId int32) (*bean.AppDeleteResponseDTO, error) { +func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConfig.Pipeline, ctx context.Context, deleteAction int, deleteFromAcd bool, foregroundDelete bool, userId int32) (*bean.AppDeleteResponseDTO, error) { cascadeDelete := true forceDelete := false deleteResponse := &bean.AppDeleteResponseDTO{ @@ -1692,8 +1693,8 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf impl.logger.Errorw("error in deleting deployment config for pipeline", "appId", pipeline.AppId, "envId", pipeline.EnvironmentId, "err", err) return nil, err } - //delete app from argo cd, if created - if pipeline.DeploymentAppCreated == true { + //delete app from argo cd/flux/helm, if created — skipped for foreground (detach) delete + if pipeline.DeploymentAppCreated == true && !foregroundDelete { deploymentAppName := pipeline.DeploymentAppName if util.IsAcdApp(envDeploymentConfig.DeploymentAppType) { if !forceDelete && !deleteResponse.ClusterReachable { @@ -1755,6 +1756,9 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf } } + } else if pipeline.DeploymentAppCreated && foregroundDelete { + impl.logger.Infow("foreground delete: skipping deployed resource deletion, detaching pipeline", + "pipelineId", pipeline.Id, "deploymentAppName", pipeline.DeploymentAppName) } err = tx.Commit() if err != nil { @@ -1762,7 +1766,7 @@ func (impl *CdPipelineConfigServiceImpl) DeleteCdPipeline(pipeline *pipelineConf return deleteResponse, err } deleteResponse.DeleteInitiated = true - impl.pipelineConfigEventPublishService.PublishCDPipelineDelete(pipeline.Id, userId) + impl.pipelineConfigEventPublishService.PublishCDPipelineDelete(pipeline.Id, userId, foregroundDelete) return deleteResponse, nil } @@ -1843,7 +1847,7 @@ func (impl *CdPipelineConfigServiceImpl) DeleteHelmTypePipelineDeploymentApp(ctx func (impl *CdPipelineConfigServiceImpl) DeleteACDAppCdPipelineWithNonCascade(pipeline *pipelineConfig.Pipeline, ctx context.Context, forceDelete bool, userId int32) error { if forceDelete { - _, err := impl.DeleteCdPipeline(pipeline, ctx, bean.FORCE_DELETE, false, userId) + _, err := impl.DeleteCdPipeline(pipeline, ctx, bean.FORCE_DELETE, false, false, userId) return err } envDeploymentConfig, err := impl.deploymentConfigService.GetConfigForDevtronApps(nil, pipeline.AppId, pipeline.EnvironmentId) @@ -2403,7 +2407,7 @@ func (impl *CdPipelineConfigServiceImpl) MarkGitOpsDevtronAppsDeletedWhereArgoAp } impl.logger.Warnw("app not found in argo, deleting from db ", "err", err) //make call to delete it from pipeline DB because it's ACD counterpart is deleted - _, err = impl.DeleteCdPipeline(pipeline, context.Background(), bean.FORCE_DELETE, false, 1) + _, err = impl.DeleteCdPipeline(pipeline, context.Background(), bean.FORCE_DELETE, false, false, 1) if err != nil { impl.logger.Errorw("error in deleting cd pipeline", "err", err) return acdAppFound, err @@ -3219,7 +3223,7 @@ func (impl *CdPipelineConfigServiceImpl) BulkDeleteCdPipelines(impactedPipelines EnvironmentName: pipeline.Environment.Name, } if !dryRun { - deleteResponse, err := impl.DeleteCdPipeline(pipeline, ctx, deleteAction, true, userId) + deleteResponse, err := impl.DeleteCdPipeline(pipeline, ctx, deleteAction, true, impl.deploymentConfig.ForegroundDeleteCdPipeline, userId) if err != nil { impl.logger.Errorw("error in deleting cd pipeline", "err", err, "pipelineId", pipeline.Id) respDto.DeletionResult = fmt.Sprintf("Not able to delete pipeline, %v", err) diff --git a/util/GlobalConfig.go b/util/GlobalConfig.go index df2c67e896..f1eabbe467 100644 --- a/util/GlobalConfig.go +++ b/util/GlobalConfig.go @@ -39,6 +39,7 @@ type DeploymentServiceTypeConfig struct { ShouldCheckNamespaceOnClone bool `env:"SHOULD_CHECK_NAMESPACE_ON_CLONE" envDefault:"false" description:"should we check if namespace exists or not while cloning app" deprecated:"false"` ValidateExtAppChart bool `env:"VALIDATE_EXT_APP_CHART_TYPE" envDefault:"false" description:"validate external flux app chart" deprecated:"false"` FeatureMigrateFluxApplicationEnable bool `env:"FEATURE_MIGRATE_FLUX_APPLICATION_ENABLE" envDefault:"false" description:"enable flux application services" deprecated:"false"` + ForegroundDeleteCdPipeline bool `env:"FOREGROUND_DELETE_CD_PIPELINE" envDefault:"false" description:"If enabled, deleting a CD pipeline will not delete the underlying deployed resource (helm release/argocd application/flux release) unless the request explicitly overrides it via payload." deprecated:"false"` } func (d *DeploymentServiceTypeConfig) IsFeatureMigrateArgoCdApplicationEnable() bool { diff --git a/wire_gen.go b/wire_gen.go index 1918fe255e..b27117cd9a 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -972,7 +972,7 @@ func InitializeApp() (*App, error) { bulkEditRepositoryImpl := repository31.NewBulkEditRepository(db, sugaredLogger) deployedAppServiceImpl := deployedApp.NewDeployedAppServiceImpl(sugaredLogger, k8sCommonServiceImpl, devtronAppsHandlerServiceImpl, environmentRepositoryImpl, pipelineRepositoryImpl, cdWorkflowRepositoryImpl) bulkUpdateServiceEntImpl := service8.NewBulkUpdateServiceEntImpl() - bulkUpdateServiceImpl := service8.NewBulkUpdateServiceImpl(bulkEditRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, handlerServiceImpl, bulkUpdateServiceEntImpl) + bulkUpdateServiceImpl := service8.NewBulkUpdateServiceImpl(bulkEditRepositoryImpl, sugaredLogger, environmentRepositoryImpl, pipelineRepositoryImpl, appRepositoryImpl, deploymentTemplateHistoryServiceImpl, configMapHistoryServiceImpl, pipelineBuilderImpl, enforcerUtilImpl, ciHandlerImpl, ciPipelineRepositoryImpl, appWorkflowRepositoryImpl, appWorkflowServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, deployedAppServiceImpl, cdPipelineEventPublishServiceImpl, handlerServiceImpl, environmentVariables, bulkUpdateServiceEntImpl) bulkUpdateRestHandlerImpl := restHandler.NewBulkUpdateRestHandlerImpl(pipelineBuilderImpl, sugaredLogger, bulkUpdateServiceImpl, chartServiceImpl, propertiesConfigServiceImpl, userServiceImpl, enforcerImpl, ciHandlerImpl, validate, clientImpl, ciPipelineRepositoryImpl, pipelineRepositoryImpl, enforcerUtilImpl, environmentServiceImpl, gitRegistryConfigImpl, dockerRegistryConfigImpl, cdHandlerImpl, appCloneServiceImpl, appWorkflowServiceImpl, materialRepositoryImpl) bulkUpdateRouterImpl := router.NewBulkUpdateRouterImpl(bulkUpdateRestHandlerImpl) webhookSecretValidatorImpl := gitWebhook.NewWebhookSecretValidatorImpl(sugaredLogger) From f3b1f5f981a3e17a552435613cec88361a654e1d Mon Sep 17 00:00:00 2001 From: SATYAsasini Date: Tue, 15 Sep 2026 14:22:34 +0530 Subject: [PATCH 08/12] chore: align migration sequence with enterprise * renumber variable_entity_mapping_index 36504600 -> 36704600 (36504600 was duplicated with add_ci_workflow_active_status_index) * add empty placeholder 36604600_approval_declined_notification (enterprise-only feature) --- scripts/sql/36604600_approval_declined_notification.down.sql | 0 scripts/sql/36604600_approval_declined_notification.up.sql | 0 ...x.down.sql => 36704600_variable_entity_mapping_index.down.sql} | 0 ...index.up.sql => 36704600_variable_entity_mapping_index.up.sql} | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 scripts/sql/36604600_approval_declined_notification.down.sql create mode 100644 scripts/sql/36604600_approval_declined_notification.up.sql rename scripts/sql/{36504600_variable_entity_mapping_index.down.sql => 36704600_variable_entity_mapping_index.down.sql} (100%) rename scripts/sql/{36504600_variable_entity_mapping_index.up.sql => 36704600_variable_entity_mapping_index.up.sql} (100%) diff --git a/scripts/sql/36604600_approval_declined_notification.down.sql b/scripts/sql/36604600_approval_declined_notification.down.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/sql/36604600_approval_declined_notification.up.sql b/scripts/sql/36604600_approval_declined_notification.up.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/sql/36504600_variable_entity_mapping_index.down.sql b/scripts/sql/36704600_variable_entity_mapping_index.down.sql similarity index 100% rename from scripts/sql/36504600_variable_entity_mapping_index.down.sql rename to scripts/sql/36704600_variable_entity_mapping_index.down.sql diff --git a/scripts/sql/36504600_variable_entity_mapping_index.up.sql b/scripts/sql/36704600_variable_entity_mapping_index.up.sql similarity index 100% rename from scripts/sql/36504600_variable_entity_mapping_index.up.sql rename to scripts/sql/36704600_variable_entity_mapping_index.up.sql From 7b773ff760a201407f0201a77568a3c77bff371c Mon Sep 17 00:00:00 2001 From: vivekanandan-devtron Date: Wed, 16 Sep 2026 13:44:28 +0530 Subject: [PATCH 09/12] misc: main sync develop 16sep (#7039) * fix(security): enforce caller RBAC in API token webhook endpoint GetAllApiTokensForWebhook checked whether each STORED api token had trigger permission on the requested project/env, but never checked whether the CALLING user was permitted to query that project/env at all. As a result any authenticated user could request an arbitrary projectName/environmentName/appName and receive the plaintext token values of every stored api token (including super-admin tokens, whose Casbin policy is "*" and therefore always passes the per-token check). Add the same enforcement pattern used by the sibling handlers in this file, but scoped to the resource being queried: verify the requesting user's own token has trigger permission on the requested project and every requested environment before the service call is made, so no token data is looked up or returned for a project/env the caller cannot access. Fixes #7013 * chore: address SonarCloud finding in GetAllApiTokensForWebhook Inline the unnecessary "ok" variable in the caller-RBAC check added in the previous commit, per SonarCloud's "remove this unnecessary variable declaration and use the expression directly" finding on PR #7016. * fix(security): don't leak more-privileged tokens to less-privileged callers Checking that a caller has trigger permission on the requested project/env was not sufficient: a super-admin api-token's Casbin policy is "*", so it always satisfies that same per-object check regardless of how narrow the actual project/env being queried is. That meant any caller with legitimate, narrow trigger access to some app/env could still have super-admin (or other more-privileged) tokens returned alongside their own in the same response. Add callerDominatesToken, which excludes a stored token from the response whenever it is a super-admin token and the caller is not, reusing the same super-admin check already used by the other handlers in this file. The per-object check now runs against each stored token before this dominance check, matching the existing per-token-per-object evaluation the service already performs. The Enforcer interface here only exposes single-tuple Enforce/ EnforceInBatch calls, not policy or role enumeration, so a fully generic "is token's permission a strict subset of the caller's" comparison across every possible object isn't available without extending that interface. Super-admin dominance is the well-defined, checkable case that matches this issue (leaking admin token values), so it's what this fix targets; a more general scope comparison would need broader RBAC changes outside this endpoint's blast radius. * chore: address SonarCloud param-grouping findings in ApiTokenRestHandler Group consecutive same-type parameters (storedToken/projObj/envObj and callerToken/storedToken) per SonarCloud's finding on the code added in the previous two commits. * fix(security): simplify to a straightforward super-admin gate Per thread decision: GetAllApiTokensForWebhook's only real caller is the dashboard's webhook config modal, which is already super-admin-only in the UI. Rather than layering a per-project/env caller check plus a token-dominance check on top of the existing per-stored-token filter, just gate the whole endpoint on the caller being a super-admin - the same Enforce(token, ResourceGlobal, ActionUpdate, "*") check already used by every other handler in this file (GetAllApiTokens, CreateApiToken, UpdateApiToken, DeleteApiToken). This removes the per-object caller check and the callerDominatesToken logic added in the previous two commits; the existing per-stored-token CheckAuthorizationForWebhook callback into the service is left as-is since it still does useful filtering (only tokens with trigger permission on the requested project/env are returned), it's just no longer load-bearing for access control now that the endpoint itself requires a super-admin caller. Net effect vs main: an 8-line super-admin check added right after GetLoggedInUser, matching the pattern already used by every sibling handler in this file. * chore: address SonarCloud finding in super-admin gate check Inline the unnecessary "ok" variable in the super-admin check added to GetAllApiTokensForWebhook, per SonarCloud's "remove this unnecessary variable declaration" finding on PR #7016. * feat: decouple app lifecycle actions; add indexes to improve CI query performance - Introduce `createApp`/`deleteApp` actions to segregate application lifecycle control from pipeline/workflow/config operations. - Update RBAC enforcement to reflect the new `createApp`/`deleteApp` actions for better granularity. - Add `idx_ci_artifact`, `idx_ci_workflow`, and other indexes to improve CI query execution and resolve PG_READ_TIMEOUT issues. - Introduce GIN index for `git_triggers` to optimize material/commit lookup in `validateBuildSequence`. * fix(user): scope filter-based bulk delete to match listing filters DELETE /orchestrator/user/bulk with a listingRequest filter (instead of explicit ids) resolved the delete target list via the same query builder the listing endpoint uses, but skipped the normalization the listing endpoint applies first (SetDefaultValuesIfNotPresent, setStatusFilterType). With no default page size and no status filtering applied, a filter such as {"searchKey":"","status":["inactive"]} resolved to every active user instead of the intended filtered subset, causing a bulk delete of far more users than intended. getUserIdsHonoringFilters now applies the same normalization the listing path (GetAllWithFilters) applies before building the query, and rejects a filter-based bulk delete that carries no concrete filter criteria instead of letting it silently resolve to "every user". The explicit ids-based delete path is unchanged. Fixes #7020 Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01J5DesTG5XvPdmPCDuL6izq * feat: resolve RBAC inconsistencies for helm apps with unique identifiers When an external helm app is linked to the chart store, app_name holds the unique identifier (--) while display_name holds the release name. RBAC policies are always created against the release name, since that is what the listing APIs expose to the user. EnforcerUtil did not account for this: - GetHelmObjectByAppNameAndEnvId looked the app up by app_name only, so a release-name lookup returned ErrNoRows and the object collapsed to "//", making the app invisible to any non super-admin user. - Both it and GetHelmObject built the object from application.AppName, i.e. the unique identifier, which never matched the granted policy. - Neither substituted the "unassigned" project when team_id is 0, so an app linked without a project produced an empty project segment. EnforcerUtilHelm.GetAppRBACNameByInstalledAppId and GetAppRBACNameByInstalledAppIdAndTeamId had the same raw AppName problem. Resolve the app the same way getAppObject already does: reconstruct the unique identifier from the environment and try that first, falling back to the plain name for regular chart store apps, legacy rows and callers that already pass the identifier. Emit the display name via the new App.GetAppNameForRbac(), and fall back to the unassigned project when no project is assigned. Fixing the primitives covers all downstream call sites without touching the handlers. Co-Authored-By: Claude Opus 5 (1M context) * Conditions saved with the plugin definition (plugin_step_condition) are intentionally not sent for the plugin's internal steps. * feat: RBAC support for external argo/flux apps (#7014) * chore: create a makefile to setup and clean resources for development * feat: add flag for frontend to show argo/flux tabs - add migration files - append flags in the /orchestrator/user/check/roles API to show/hide argo/flux tabs * feat: changes for dropdown API in user permissions page * fix: missing dependencies in constructor method * fix: improper filtering logic - earlier filter was only applied with the cluster ID - add changes to filter based on cluster ID's and namespaces * fix: prevent connecting to kubelink for failed clusters * wip: policy enforcements * wip: policy enforcements * chore: remove test folder * fix: code review fixes * fix: code review fixes * fix: code review fixes * chore: rename migration scripts to maintain sync with enterprise * chore: change migration script number * fix: roles missing while fetching user roles * fix: deleted roles being listed * fix: deleted roles being listed * fix: review fixes * chore: rename migration files to correct sequence --------- Co-authored-by: Claude Co-authored-by: Shivam Nagar <124123645+Shivam-nagar23@users.noreply.github.com> Co-authored-by: Shivam-nagar23 Co-authored-by: Prakash Kumar --- api/apiToken/ApiTokenRestHandler.go | 10 +- .../ArgoApplicationRestHandler.go | 50 +++- api/auth/user/UserRestHandler.go | 33 ++- .../FluxApplicationRestHandler.go | 29 ++- api/helm-app/wire_helmApp.go | 3 + .../application/k8sApplicationRestHandler.go | 77 ++++-- api/restHandler/CoreAppRestHandler.go | 3 +- .../configure/PipelineConfigRestHandler.go | 29 ++- api/userResource/bean/bean.go | 2 + cmd/external-app/wire_gen.go | 9 +- internal/sql/repository/app/AppRepository.go | 12 + pkg/auth/authorisation/casbin/rbacpolicy.go | 11 + pkg/auth/user/UserCommonService.go | 29 ++- pkg/auth/user/UserService.go | 9 + .../UserService_bulkdelete_filter_test.go | 100 ++++++++ pkg/auth/user/UserService_ent.go | 7 + pkg/auth/user/bean/bean.go | 2 + .../user/repository/UserAuthRepository.go | 52 +++- pkg/fluxApplication/FluxApplicationService.go | 152 ++++++++---- pkg/k8s/application/k8sApplicationService.go | 10 +- .../PipelineStagePipelineConditions_test.go | 75 ++++++ .../PipelineStagePluginConditions_test.go | 230 ++++++++++++++++++ pkg/pipeline/PipelineStageService.go | 47 ++-- .../UserResourceExtendedService.go | 11 +- .../UserResourceRbacExtendedService.go | 18 ++ pkg/userResource/UserResourceRbacService.go | 7 +- pkg/userResource/UserResourceService.go | 88 +++++-- pkg/userResource/adapter/adapter.go | 32 +++ pkg/userResource/bean/bean.go | 39 ++- pkg/userResource/bean/messages.go | 5 +- pkg/userResource/helper/helper.go | 52 +++- pkg/userResource/logicRouteService.go | 25 +- scripts/casbin/12_argo_flux_rbac.down.sql | 2 + scripts/casbin/12_argo_flux_rbac.up.sql | 3 + .../36504600_dev_admin_app_lifecycle.down.sql | 5 + .../36504600_dev_admin_app_lifecycle.up.sql | 13 + scripts/sql/36804600_argo_flux_rbac.down.sql | 2 + scripts/sql/36804600_argo_flux_rbac.up.sql | 111 +++++++++ util/rbac/EnforcerUtil.go | 72 ++++-- util/rbac/EnforcerUtilGitOps.go | 74 ++++++ util/rbac/EnforcerUtilHelm.go | 8 +- util/rbac/EnforcerUtilHelmObject_test.go | 212 ++++++++++++++++ wire_gen.go | 9 +- 43 files changed, 1565 insertions(+), 204 deletions(-) create mode 100644 pkg/auth/user/UserService_bulkdelete_filter_test.go create mode 100644 pkg/pipeline/PipelineStagePipelineConditions_test.go create mode 100644 pkg/pipeline/PipelineStagePluginConditions_test.go create mode 100644 scripts/casbin/12_argo_flux_rbac.down.sql create mode 100644 scripts/casbin/12_argo_flux_rbac.up.sql create mode 100644 scripts/sql/36504600_dev_admin_app_lifecycle.down.sql create mode 100644 scripts/sql/36504600_dev_admin_app_lifecycle.up.sql create mode 100644 scripts/sql/36804600_argo_flux_rbac.down.sql create mode 100644 scripts/sql/36804600_argo_flux_rbac.up.sql create mode 100644 util/rbac/EnforcerUtilGitOps.go create mode 100644 util/rbac/EnforcerUtilHelmObject_test.go diff --git a/api/apiToken/ApiTokenRestHandler.go b/api/apiToken/ApiTokenRestHandler.go index d003385d98..b4918900e5 100644 --- a/api/apiToken/ApiTokenRestHandler.go +++ b/api/apiToken/ApiTokenRestHandler.go @@ -220,7 +220,15 @@ func (impl ApiTokenRestHandlerImpl) GetAllApiTokensForWebhook(w http.ResponseWri return } - // handle super-admin RBAC + // handle super-admin RBAC - this endpoint's only caller is the dashboard's + // webhook config modal, which is itself super-admin-only, so gate the + // whole endpoint the same way the other handlers in this file do. + token := r.Header.Get("token") + if !impl.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*") { + common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) + return + } + v := r.URL.Query() projectName := v.Get("projectName") environmentName := v.Get("environmentName") diff --git a/api/argoApplication/ArgoApplicationRestHandler.go b/api/argoApplication/ArgoApplicationRestHandler.go index edcbabdca9..5b816db2cd 100644 --- a/api/argoApplication/ArgoApplicationRestHandler.go +++ b/api/argoApplication/ArgoApplicationRestHandler.go @@ -21,8 +21,10 @@ import ( "errors" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/argoApplication" + "github.com/devtron-labs/devtron/pkg/argoApplication/bean" "github.com/devtron-labs/devtron/pkg/argoApplication/read" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" + "github.com/devtron-labs/devtron/util/rbac" "go.uber.org/zap" "net/http" "strconv" @@ -39,26 +41,24 @@ type ArgoApplicationRestHandlerImpl struct { readService read.ArgoApplicationReadService logger *zap.SugaredLogger enforcer casbin.Enforcer + enforcerUtilGitOps rbac.EnforcerUtilGitOps } func NewArgoApplicationRestHandlerImpl(argoApplicationService argoApplication.ArgoApplicationService, - readService read.ArgoApplicationReadService, logger *zap.SugaredLogger, enforcer casbin.Enforcer) *ArgoApplicationRestHandlerImpl { + readService read.ArgoApplicationReadService, logger *zap.SugaredLogger, enforcer casbin.Enforcer, + enforcerUtilGitOps rbac.EnforcerUtilGitOps) *ArgoApplicationRestHandlerImpl { return &ArgoApplicationRestHandlerImpl{ argoApplicationService: argoApplicationService, readService: readService, logger: logger, enforcer: enforcer, + enforcerUtilGitOps: enforcerUtilGitOps, } } func (handler *ArgoApplicationRestHandlerImpl) ListApplications(w http.ResponseWriter, r *http.Request) { - // handle super-admin RBAC token := r.Header.Get("token") - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { - common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) - return - } v := r.URL.Query() clusterIdString := v.Get("clusterIds") var clusterIds []int @@ -80,16 +80,34 @@ func (handler *ArgoApplicationRestHandlerImpl) ListApplications(w http.ResponseW common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return } - common.WriteJsonResp(w, nil, resp, http.StatusOK) + // RBAC enforcer applying: filter the listing to the applications the caller may see. + // Batched rather than a per-app Enforce loop; an app whose object cannot be built is dropped. + objects := make([]string, 0, len(resp)) + objectByApp := make(map[*bean.ArgoApplicationListDto]string, len(resp)) + for _, app := range resp { + object := handler.enforcerUtilGitOps.GetExternalGitOpsAppObjectByClusterName(app.ClusterName, app.Namespace, app.Name) + if len(object) == 0 { + continue + } + objectByApp[app] = object + objects = append(objects, object) + } + authorisedObjects := make(map[string]bool) + if len(objects) > 0 { + authorisedObjects = handler.enforcer.EnforceInBatch(token, casbin.ResourceArgoApp, casbin.ActionGet, objects) + } + authorisedApps := make([]*bean.ArgoApplicationListDto, 0, len(resp)) + for _, app := range resp { + if object, ok := objectByApp[app]; ok && authorisedObjects[strings.ToLower(object)] { + authorisedApps = append(authorisedApps, app) + } + } + //RBAC enforcer Ends + common.WriteJsonResp(w, nil, authorisedApps, http.StatusOK) } func (handler *ArgoApplicationRestHandlerImpl) GetApplicationDetail(w http.ResponseWriter, r *http.Request) { - // handle super-admin RBAC token := r.Header.Get("token") - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { - common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) - return - } ctx := r.Context() ctx = context.WithValue(ctx, "token", token) @@ -108,6 +126,14 @@ func (handler *ArgoApplicationRestHandlerImpl) GetApplicationDetail(w http.Respo return } } + // RBAC enforcer applying + object := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(clusterId, namespace, resourceName) + if len(object) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceArgoApp, casbin.ActionGet, object) { + common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) + return + } + //RBAC enforcer Ends + resp, err := handler.readService.GetAppDetailEA(ctx, resourceName, namespace, clusterId) if err != nil { handler.logger.Errorw("error in getting argo application app detail", "err", err, "resourceName", resourceName, "clusterId", clusterId) diff --git a/api/auth/user/UserRestHandler.go b/api/auth/user/UserRestHandler.go index 37188c14bd..8e617250b3 100644 --- a/api/auth/user/UserRestHandler.go +++ b/api/auth/user/UserRestHandler.go @@ -19,13 +19,14 @@ package user import ( "encoding/json" "errors" + "net/http" + "strconv" + "strings" + util2 "github.com/devtron-labs/devtron/api/auth/user/util" "github.com/devtron-labs/devtron/pkg/auth/user/helper" "github.com/devtron-labs/devtron/util/commonEnforcementFunctionsUtil" "github.com/gorilla/schema" - "net/http" - "strconv" - "strings" "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/internal/util" @@ -38,6 +39,16 @@ import ( "gopkg.in/go-playground/validator.v9" ) +const ( + resultKeyArgoAppAccess = "hasArgoAppAccess" + resultKeyFluxAppAccess = "hasFluxAppAccess" +) + +var gitOpsAccessResultKeyByResource = map[string]string{ + casbin.ResourceArgoApp: resultKeyArgoAppAccess, + casbin.ResourceFluxApp: resultKeyFluxAppAccess, +} + type UserRestHandler interface { CreateUser(w http.ResponseWriter, r *http.Request) UpdateUser(w http.ResponseWriter, r *http.Request) @@ -795,9 +806,25 @@ func (handler UserRestHandlerImpl) CheckUserRoles(w http.ResponseWriter, r *http result := make(map[string]interface{}) result["roles"] = roles result["superAdmin"] = false + result[resultKeyArgoAppAccess] = false + result[resultKeyFluxAppAccess] = false for _, item := range roles { if item == bean2.SUPERADMIN { result["superAdmin"] = true + result[resultKeyArgoAppAccess] = true + result[resultKeyFluxAppAccess] = true + continue + } + + roleFragments := strings.Split(item, "_") + resourceActionFragment := strings.Split(roleFragments[0], ":") + + if len(resourceActionFragment) < 2 { + continue + } + + if resultKey, ok := gitOpsAccessResultKeyByResource[resourceActionFragment[0]]; ok { + result[resultKey] = true } } common.WriteJsonResp(w, err, result, http.StatusOK) diff --git a/api/fluxApplication/FluxApplicationRestHandler.go b/api/fluxApplication/FluxApplicationRestHandler.go index f1ef3d616c..bc1b9c3c02 100644 --- a/api/fluxApplication/FluxApplicationRestHandler.go +++ b/api/fluxApplication/FluxApplicationRestHandler.go @@ -5,6 +5,7 @@ import ( "github.com/devtron-labs/devtron/api/restHandler/common" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" clientErrors "github.com/devtron-labs/devtron/pkg/errors" + "github.com/devtron-labs/devtron/util/rbac" "github.com/devtron-labs/devtron/pkg/fluxApplication" "github.com/gorilla/mux" "go.uber.org/zap" @@ -20,26 +21,34 @@ type FluxApplicationRestHandlerImpl struct { fluxApplicationService fluxApplication.FluxApplicationService logger *zap.SugaredLogger enforcer casbin.Enforcer + enforcerUtilGitOps rbac.EnforcerUtilGitOps } func NewFluxApplicationRestHandlerImpl(fluxApplicationService fluxApplication.FluxApplicationService, - logger *zap.SugaredLogger, enforcer casbin.Enforcer) *FluxApplicationRestHandlerImpl { + logger *zap.SugaredLogger, enforcer casbin.Enforcer, + enforcerUtilGitOps rbac.EnforcerUtilGitOps) *FluxApplicationRestHandlerImpl { return &FluxApplicationRestHandlerImpl{ fluxApplicationService: fluxApplicationService, logger: logger, enforcer: enforcer, + enforcerUtilGitOps: enforcerUtilGitOps, } } +// checkFluxAppAuth builds the RBAC object from the app identity and enforces on it. Passed into +// the service because the app list is streamed and cannot be filtered after the fact. +func (handler *FluxApplicationRestHandlerImpl) checkFluxAppAuth(token string, clusterName string, namespace string, appName string) bool { + object := handler.enforcerUtilGitOps.GetExternalGitOpsAppObjectByClusterName(clusterName, namespace, appName) + if len(object) == 0 { + return false + } + return handler.enforcer.Enforce(token, casbin.ResourceFluxApp, casbin.ActionGet, object) +} + func (handler *FluxApplicationRestHandlerImpl) ListFluxApplications(w http.ResponseWriter, r *http.Request) { - //handle super-admin RBAC token := r.Header.Get("token") - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { - common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) - return - } v := r.URL.Query() clusterIdString := v.Get("clusterIds") var clusterIds []int @@ -59,7 +68,7 @@ func (handler *FluxApplicationRestHandlerImpl) ListFluxApplications(w http.Respo return } handler.logger.Debugw("extracted ClusterIds successfully ", "clusterIds", clusterIds) - handler.fluxApplicationService.ListFluxApplications(r.Context(), clusterIds, noStream, w) + handler.fluxApplicationService.ListFluxApplications(r.Context(), clusterIds, noStream, w, token, handler.checkFluxAppAuth) } func (handler *FluxApplicationRestHandlerImpl) GetApplicationDetail(w http.ResponseWriter, r *http.Request) { @@ -76,12 +85,14 @@ func (handler *FluxApplicationRestHandlerImpl) GetApplicationDetail(w http.Respo return } - // handle super-admin RBAC + // RBAC enforcer applying token := r.Header.Get("token") - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + object := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.Name) + if len(object) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceFluxApp, casbin.ActionGet, object) { common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) return } + //RBAC enforcer Ends res, err := handler.fluxApplicationService.GetFluxAppDetail(r.Context(), appIdentifier) if err != nil { diff --git a/api/helm-app/wire_helmApp.go b/api/helm-app/wire_helmApp.go index 86f45e672c..af668990bb 100644 --- a/api/helm-app/wire_helmApp.go +++ b/api/helm-app/wire_helmApp.go @@ -41,4 +41,7 @@ var HelmAppWireSet = wire.NewSet( gRPC.GetConfig, rbac.NewEnforcerUtilHelmImpl, wire.Bind(new(rbac.EnforcerUtilHelm), new(*rbac.EnforcerUtilHelmImpl)), + + rbac.NewEnforcerUtilGitOpsImpl, + wire.Bind(new(rbac.EnforcerUtilGitOps), new(*rbac.EnforcerUtilGitOpsImpl)), ) diff --git a/api/k8s/application/k8sApplicationRestHandler.go b/api/k8s/application/k8sApplicationRestHandler.go index a3d64a7ff5..8417225c8e 100644 --- a/api/k8s/application/k8sApplicationRestHandler.go +++ b/api/k8s/application/k8sApplicationRestHandler.go @@ -23,6 +23,13 @@ import ( "encoding/json" "errors" "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + "time" + "github.com/devtron-labs/common-lib/utils" util3 "github.com/devtron-labs/common-lib/utils/k8s" k8sCommonBean "github.com/devtron-labs/common-lib/utils/k8s/commonBean" @@ -52,14 +59,8 @@ import ( errors2 "github.com/juju/errors" "go.uber.org/zap" "gopkg.in/go-playground/validator.v9" - "io" errors3 "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "net/http" - "regexp" - "strconv" - "strings" - "time" ) type K8sApplicationRestHandler interface { @@ -91,6 +92,7 @@ type K8sApplicationRestHandlerImpl struct { validator *validator.Validate enforcerUtil rbac.EnforcerUtil enforcerUtilHelm rbac.EnforcerUtilHelm + enforcerUtilGitOps rbac.EnforcerUtilGitOps helmAppService client.HelmAppService userService user.UserService k8sCommonService k8s.K8sCommonService @@ -99,7 +101,7 @@ type K8sApplicationRestHandlerImpl struct { argoApplicationReadService read.ArgoApplicationReadService } -func NewK8sApplicationRestHandlerImpl(logger *zap.SugaredLogger, k8sApplicationService application2.K8sApplicationService, pump connector.Pump, terminalSessionHandler terminal.TerminalSessionHandler, enforcer casbin.Enforcer, enforcerUtilHelm rbac.EnforcerUtilHelm, enforcerUtil rbac.EnforcerUtil, helmAppService client.HelmAppService, userService user.UserService, k8sCommonService k8s.K8sCommonService, validator *validator.Validate, envVariables *util.EnvironmentVariables, fluxAppService fluxApplication.FluxApplicationService, argoApplicationReadService read.ArgoApplicationReadService, +func NewK8sApplicationRestHandlerImpl(logger *zap.SugaredLogger, k8sApplicationService application2.K8sApplicationService, pump connector.Pump, terminalSessionHandler terminal.TerminalSessionHandler, enforcer casbin.Enforcer, enforcerUtilHelm rbac.EnforcerUtilHelm, enforcerUtilGitOps rbac.EnforcerUtilGitOps, enforcerUtil rbac.EnforcerUtil, helmAppService client.HelmAppService, userService user.UserService, k8sCommonService k8s.K8sCommonService, validator *validator.Validate, envVariables *util.EnvironmentVariables, fluxAppService fluxApplication.FluxApplicationService, argoApplicationReadService read.ArgoApplicationReadService, ) *K8sApplicationRestHandlerImpl { return &K8sApplicationRestHandlerImpl{ logger: logger, @@ -109,6 +111,7 @@ func NewK8sApplicationRestHandlerImpl(logger *zap.SugaredLogger, k8sApplicationS enforcer: enforcer, validator: validator, enforcerUtilHelm: enforcerUtilHelm, + enforcerUtilGitOps: enforcerUtilGitOps, enforcerUtil: enforcerUtil, helmAppService: helmAppService, userService: userService, @@ -207,7 +210,20 @@ func (handler *K8sApplicationRestHandlerImpl) GetResource(w http.ResponseWriter, canUpdate := false // Obfuscate secret if user does not have edit access - if request.AppIdentifier == nil && request.DevtronAppIdentifier == nil && request.AppType != bean2.ArgoAppType && request.ClusterId > 0 { // if the appType is not argoAppType,then verify logic w.r.t resource browser, when rbac for argoApp is introduced, handle rbac accordingly + if request.AppType == bean2.ArgoAppType && request.ExternalArgoAppIdentifier != nil { + // External Argo app: edit access is decided by the app-level permission, not by the + // resource-browser cluster entity. Without this an Argo admin would never see Secret + // values, because canUpdate would stay false and the manifest would be masked below. + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(request.ExternalArgoAppIdentifier.ClusterId, + request.ExternalArgoAppIdentifier.Namespace, request.ExternalArgoAppIdentifier.AppName) + canUpdate = len(rbacObject) > 0 && + handler.enforcer.Enforce(token, casbin.ResourceArgoApp, casbin.ActionUpdate, rbacObject) + } else if request.AppType == bean2.FluxAppType && request.ExternalFluxAppIdentifier != nil { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(request.ExternalFluxAppIdentifier.ClusterId, + request.ExternalFluxAppIdentifier.Namespace, request.ExternalFluxAppIdentifier.Name) + canUpdate = len(rbacObject) > 0 && + handler.enforcer.Enforce(token, casbin.ResourceFluxApp, casbin.ActionUpdate, rbacObject) + } else if request.AppIdentifier == nil && request.DevtronAppIdentifier == nil && request.AppType != bean2.ArgoAppType && request.ClusterId > 0 { // if the appType is not argoAppType,then verify logic w.r.t resource browser // Verify update access for Resource Browser canUpdate = handler.k8sApplicationService.ValidateClusterResourceBean(r.Context(), request.ClusterId, resource.ManifestResponse.Manifest, request.K8sRequest.ResourceIdentifier.GroupVersionKind, handler.getRbacCallbackForResource(token, casbin.ActionUpdate)) if !canUpdate { @@ -299,7 +315,8 @@ func (handler *K8sApplicationRestHandlerImpl) GetHostUrlsByBatch(w http.Response return } // RBAC enforcer applying - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.AppName) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceArgoApp, casbin.ActionGet, rbacObject) { common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) return } @@ -326,7 +343,8 @@ func (handler *K8sApplicationRestHandlerImpl) GetHostUrlsByBatch(w http.Response return } // RBAC enforcer applying - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.Name) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceFluxApp, casbin.ActionGet, rbacObject) { common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) return } @@ -718,7 +736,9 @@ func (handler *K8sApplicationRestHandlerImpl) requestValidationAndRBAC(w http.Re return } //RBAC enforcer starts here - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(request.ExternalFluxAppIdentifier.ClusterId, + request.ExternalFluxAppIdentifier.Namespace, request.ExternalFluxAppIdentifier.Name) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceFluxApp, casbin.ActionGet, rbacObject) { common.WriteJsonResp(w, errors2.New("unauthorized"), nil, http.StatusForbidden) return } @@ -738,7 +758,8 @@ func (handler *K8sApplicationRestHandlerImpl) requestValidationAndRBAC(w http.Re } //RBAC enforcer starts here - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.AppName) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceArgoApp, casbin.ActionGet, rbacObject) { common.WriteJsonResp(w, errors2.New("unauthorized"), nil, http.StatusForbidden) return } @@ -803,7 +824,9 @@ func (handler *K8sApplicationRestHandlerImpl) GetTerminalSession(w http.Response //RBAC enforcer Ends } else if resourceRequestBean.ExternalFluxAppIdentifier != nil { // RBAC enforcer applying For external flux app - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*"); !ok { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(resourceRequestBean.ExternalFluxAppIdentifier.ClusterId, + resourceRequestBean.ExternalFluxAppIdentifier.Namespace, resourceRequestBean.ExternalFluxAppIdentifier.Name) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceFluxApp, casbin.ActionUpdate, rbacObject) { common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) return } @@ -811,7 +834,13 @@ func (handler *K8sApplicationRestHandlerImpl) GetTerminalSession(w http.Response } else if resourceRequestBean.ExternalArgoApplicationName != "" { // RBAC enforcer applying For external Argo app - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*"); !ok { + if resourceRequestBean.ExternalArgoAppIdentifier == nil { + common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) + return + } + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(resourceRequestBean.ExternalArgoAppIdentifier.ClusterId, + resourceRequestBean.ExternalArgoAppIdentifier.Namespace, resourceRequestBean.ExternalArgoAppIdentifier.AppName) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceArgoApp, casbin.ActionUpdate, rbacObject) { common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden) return } @@ -1125,14 +1154,18 @@ func (handler *K8sApplicationRestHandlerImpl) handleEphemeralRBAC(podName, names //RBAC enforcer Ends } else if resourceRequestBean.ExternalFluxAppIdentifier != nil { //RBAC enforcer starts here - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(resourceRequestBean.ExternalFluxAppIdentifier.ClusterId, + resourceRequestBean.ExternalFluxAppIdentifier.Namespace, resourceRequestBean.ExternalFluxAppIdentifier.Name) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceFluxApp, casbin.ActionUpdate, rbacObject) { common.WriteJsonResp(w, errors2.New("unauthorized"), nil, http.StatusForbidden) return resourceRequestBean } //RBAC enforcer ends here } else if resourceRequestBean.ExternalArgoApplicationName != "" { //RBAC enforcer starts here - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionGet, "*"); !ok { + rbacObject := handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(resourceRequestBean.ExternalArgoAppIdentifier.ClusterId, + resourceRequestBean.ExternalArgoAppIdentifier.Namespace, resourceRequestBean.ExternalArgoAppIdentifier.AppName) + if len(rbacObject) == 0 || !handler.enforcer.Enforce(token, casbin.ResourceArgoApp, casbin.ActionUpdate, rbacObject) { common.WriteJsonResp(w, errors2.New("unauthorized"), nil, http.StatusForbidden) return resourceRequestBean } @@ -1179,7 +1212,11 @@ func (handler *K8sApplicationRestHandlerImpl) verifyRbacForAppRequests(token str return false, err } //RBAC enforcer starts here - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, actionType, "*"); !ok { + rbacObject = handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(argoAppIdentifier.ClusterId, argoAppIdentifier.Namespace, argoAppIdentifier.AppName) + if len(rbacObject) == 0 { + return false, nil + } + if ok := handler.enforcer.Enforce(token, casbin.ResourceArgoApp, actionType, rbacObject); !ok { return false, nil } return true, nil @@ -1247,7 +1284,11 @@ func (handler *K8sApplicationRestHandlerImpl) verifyRbacForAppRequests(token str return false, err } //RBAC enforcer starts here - if ok := handler.enforcer.Enforce(token, casbin.ResourceGlobal, actionType, "*"); !ok { + rbacObject = handler.enforcerUtilGitOps.GetExternalGitOpsAppObject(appIdentifier.ClusterId, appIdentifier.Namespace, appIdentifier.Name) + if len(rbacObject) == 0 { + return false, nil + } + if ok := handler.enforcer.Enforce(token, casbin.ResourceFluxApp, actionType, rbacObject); !ok { return false, nil } return true, nil diff --git a/api/restHandler/CoreAppRestHandler.go b/api/restHandler/CoreAppRestHandler.go index 2f0e6286c5..bbe991a5b3 100644 --- a/api/restHandler/CoreAppRestHandler.go +++ b/api/restHandler/CoreAppRestHandler.go @@ -295,7 +295,8 @@ func (handler CoreAppRestHandlerImpl) CreateApp(w http.ResponseWriter, r *http.R return } // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) - if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, fmt.Sprintf("%s/%s", team.Name, "*")); !ok { + // v1beta1 application API creates devtron apps only, so it uses the dedicated app-lifecycle action. + if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreateApp, fmt.Sprintf("%s/%s", team.Name, "*")); !ok { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) return } diff --git a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go index ab8a097f6f..e683c2cfcb 100644 --- a/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go +++ b/api/restHandler/app/pipeline/configure/PipelineConfigRestHandler.go @@ -57,6 +57,7 @@ import ( "go.opentelemetry.io/otel" "github.com/devtron-labs/devtron/internal/sql/repository" + "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/internal/sql/repository/pipelineConfig" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/appClone" @@ -265,7 +266,22 @@ func (handler *PipelineConfigRestHandlerImpl) DeleteApp(w http.ResponseWriter, r return } resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(appId) - ok := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, resourceObject, casbin.ActionDelete) + app, err := handler.pipelineBuilder.GetApp(appId) + if err != nil { + handler.Logger.Errorw("service err, GetApp", "err", err, "appId", appId) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + var ok bool + if app.AppType == helper.Job { + // app type is known to be a job, so enforce the jobs resource ONLY (not the applications OR), + // otherwise an applications:delete holder could delete the job entity, which is itself an app record + ok = handler.enforcer.Enforce(token, casbin.ResourceJobs, casbin.ActionDelete, resourceObject) + } else { + // devtron apps use the dedicated app-lifecycle action so the app-entity delete can be + // restricted independently of pipeline/workflow delete (which stay on ActionDelete) + ok = handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionDeleteApp, resourceObject) + } if !ok { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) return @@ -385,7 +401,16 @@ func (handler *PipelineConfigRestHandlerImpl) CreateApp(w http.ResponseWriter, r // with admin roles, you have to access for all the apps of the project to create new app. (admin or manager with specific app permission can't create app.) object := fmt.Sprintf("%s/%s", project.Name, "*") - isAuthorised := handler.enforcerUtil.CheckAppRbacForAppOrJob(token, object, casbin.ActionCreate) + var isAuthorised bool + if createRequest.AppType == helper.Job { + // app type is known to be a job, so enforce the jobs resource ONLY (not the applications OR), + // otherwise an applications:create holder could create the job entity, which is itself an app record + isAuthorised = handler.enforcer.Enforce(token, casbin.ResourceJobs, casbin.ActionCreate, object) + } else { + // devtron apps use the dedicated app-lifecycle action so the app-entity create can be + // restricted independently of pipeline/workflow/config creation (which stay on ActionCreate) + isAuthorised = handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreateApp, object) + } if !isAuthorised { common.WriteJsonResp(w, err, "Unauthorized User", http.StatusForbidden) return diff --git a/api/userResource/bean/bean.go b/api/userResource/bean/bean.go index b6f5c9ea2f..375b6de21a 100644 --- a/api/userResource/bean/bean.go +++ b/api/userResource/bean/bean.go @@ -31,6 +31,8 @@ type AppAndJobReqDto struct { } type ClusterReqDto struct { *bean3.ResourceRequestBean + ClusterIds []int `json:"clusterIds,omitempty"` + EnvironmentIdentifiers []string `json:"environmentIdentifiers,omitempty"` } type JobWorkflowReqDto struct { *bean2.WorkflowNamesRequest diff --git a/cmd/external-app/wire_gen.go b/cmd/external-app/wire_gen.go index 2edf485c1b..658246eeaa 100644 --- a/cmd/external-app/wire_gen.go +++ b/cmd/external-app/wire_gen.go @@ -384,8 +384,9 @@ func InitializeApp() (*App, error) { environmentReadServiceImpl := read8.NewEnvironmentReadServiceImpl(sugaredLogger, environmentRepositoryImpl) environmentRestHandlerImpl := cluster2.NewEnvironmentRestHandlerImpl(environmentServiceImpl, environmentReadServiceImpl, sugaredLogger, userServiceImpl, validate, enforcerImpl, deleteServiceImpl, k8sServiceImpl, k8sCommonServiceImpl, commonEnforcementUtilImpl) environmentRouterImpl := cluster2.NewEnvironmentRouterImpl(environmentRestHandlerImpl) + enforcerUtilGitOpsImpl := rbac.NewEnforcerUtilGitOpsImpl(sugaredLogger, clusterReadServiceImpl) argoApplicationReadServiceImpl := read9.NewArgoApplicationReadServiceImpl(sugaredLogger, clusterRepositoryImpl, k8sServiceImpl, helmAppClientImpl, helmAppServiceImpl) - k8sApplicationRestHandlerImpl := application2.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate, environmentVariables, fluxApplicationServiceImpl, argoApplicationReadServiceImpl) + k8sApplicationRestHandlerImpl := application2.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilGitOpsImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate, environmentVariables, fluxApplicationServiceImpl, argoApplicationReadServiceImpl) k8sApplicationRouterImpl := application2.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) chartRepositoryRestHandlerImpl := chartRepo2.NewChartRepositoryRestHandlerImpl(sugaredLogger, userServiceImpl, chartRepositoryServiceImpl, enforcerImpl, validate, deleteServiceImpl, attributesServiceImpl) chartRepositoryRouterImpl := chartRepo2.NewChartRepositoryRouterImpl(chartRepositoryRestHandlerImpl) @@ -505,11 +506,11 @@ func InitializeApp() (*App, error) { rbacRoleServiceImpl := user.NewRbacRoleServiceImpl(sugaredLogger, rbacRoleDataRepositoryImpl) rbacRoleRestHandlerImpl := user2.NewRbacRoleHandlerImpl(sugaredLogger, validate, rbacRoleServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl) rbacRoleRouterImpl := user2.NewRbacRoleRouterImpl(sugaredLogger, validate, rbacRoleRestHandlerImpl) - argoApplicationRestHandlerImpl := argoApplication2.NewArgoApplicationRestHandlerImpl(argoApplicationServiceImpl, argoApplicationReadServiceImpl, sugaredLogger, enforcerImpl) + argoApplicationRestHandlerImpl := argoApplication2.NewArgoApplicationRestHandlerImpl(argoApplicationServiceImpl, argoApplicationReadServiceImpl, sugaredLogger, enforcerImpl, enforcerUtilGitOpsImpl) argoApplicationRouterImpl := argoApplication2.NewArgoApplicationRouterImpl(argoApplicationRestHandlerImpl) - fluxApplicationRestHandlerImpl := fluxApplication2.NewFluxApplicationRestHandlerImpl(fluxApplicationServiceImpl, sugaredLogger, enforcerImpl) + fluxApplicationRestHandlerImpl := fluxApplication2.NewFluxApplicationRestHandlerImpl(fluxApplicationServiceImpl, sugaredLogger, enforcerImpl, enforcerUtilGitOpsImpl) fluxApplicationRouterImpl := fluxApplication2.NewFluxApplicationRouterImpl(fluxApplicationRestHandlerImpl) - userResourceServiceImpl := userResource.NewUserResourceServiceImpl(sugaredLogger, teamServiceImpl, environmentServiceImpl, clusterServiceImpl, k8sApplicationServiceImpl, enforcerUtilImpl, commonEnforcementUtilImpl, enforcerImpl, appCrudOperationServiceImpl) + userResourceServiceImpl := userResource.NewUserResourceServiceImpl(sugaredLogger, teamServiceImpl, environmentServiceImpl, clusterServiceImpl, k8sApplicationServiceImpl, enforcerUtilImpl, commonEnforcementUtilImpl, enforcerImpl, appCrudOperationServiceImpl, argoApplicationServiceImpl, fluxApplicationServiceImpl) restHandlerImpl := userResource2.NewUserResourceRestHandler(sugaredLogger, userServiceImpl, userResourceServiceImpl) routerImpl := userResource2.NewUserResourceRouterImpl(restHandlerImpl) clusterOverviewConfig, err := config4.GetClusterOverviewConfig() diff --git a/internal/sql/repository/app/AppRepository.go b/internal/sql/repository/app/AppRepository.go index d237ec8f7a..538490fbc9 100644 --- a/internal/sql/repository/app/AppRepository.go +++ b/internal/sql/repository/app/AppRepository.go @@ -53,6 +53,18 @@ func (app *App) IsAppJobOrExternalType() bool { return len(app.DisplayName) > 0 } +// GetAppNameForRbac returns the app name that rbac objects are built with. +// For an external helm app linked to chart store, app_name holds the unique identifier +// (--) while display_name holds the release name. Rbac policies are +// always created against the release name, as that is what is shown to the user, so display_name must +// win whenever it is set. +func (app *App) GetAppNameForRbac() string { + if len(app.DisplayName) > 0 { + return app.DisplayName + } + return app.AppName +} + type AppRepository interface { SaveWithTxn(pipelineGroup *App, tx *pg.Tx) error Update(app *App) error diff --git a/pkg/auth/authorisation/casbin/rbacpolicy.go b/pkg/auth/authorisation/casbin/rbacpolicy.go index d61234b868..1bca4a6418 100644 --- a/pkg/auth/authorisation/casbin/rbacpolicy.go +++ b/pkg/auth/authorisation/casbin/rbacpolicy.go @@ -45,6 +45,12 @@ const ( ResourceAdmin = "admin" ResourceGlobal = "global-resource" ResourceHelmApp = "helm-app" + + // ResourceArgoApp, ResourceFluxApp are used for app-level RBAC on external + // Argo CD / Flux CD applications. Object shape is __/. + ResourceArgoApp = "argo-app" + ResourceFluxApp = "flux-app" + ActionGet = "get" ActionCreate = "create" ActionUpdate = "update" @@ -53,6 +59,11 @@ const ( ActionTrigger = "trigger" ActionNotify = "notify" ActionExec = "exec" + // ActionCreateApp and ActionDeleteApp gate the application-entity lifecycle (create/delete an app), + // decoupled from ActionCreate/ActionDelete which also gate pipeline/workflow/config operations. + // admin/manager/super-admin keep app lifecycle via their wildcard (`*`) action policies. + ActionCreateApp = "createApp" + ActionDeleteApp = "deleteApp" ClusterResourceRegex = "%s/%s" // {cluster}/{namespace} ClusterObjectRegex = "%s/%s/%s" // {groupName}/{kindName}/{objectName} diff --git a/pkg/auth/user/UserCommonService.go b/pkg/auth/user/UserCommonService.go index 9f107d898b..17878833e5 100644 --- a/pkg/auth/user/UserCommonService.go +++ b/pkg/auth/user/UserCommonService.go @@ -18,13 +18,14 @@ package user import ( "fmt" + "math" + "strings" + "time" + bean3 "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin/bean" "github.com/devtron-labs/devtron/pkg/auth/user/adapter" "github.com/devtron-labs/devtron/pkg/auth/user/repository/bean" "golang.org/x/exp/maps" - "math" - "strings" - "time" "github.com/caarlos0/env/v6" "github.com/devtron-labs/authenticator/middleware" @@ -372,6 +373,8 @@ func (impl UserCommonServiceImpl) RemoveRolesAndReturnEliminatedPolicies(userInf if role, ok := roleIdVsRoleMap[userRoleModel.RoleId]; ok { isValidAuth := impl.checkRbacForARole(role, token, managerAuth) if !isValidAuth { + impl.logger.Warnw("not authorised to delete role, skipping", "roleId", role.Id, + "entity", role.Entity, "accessType", role.AccessType, "action", role.Action) continue } toBeDeletedUserRolesIds = append(toBeDeletedUserRolesIds, userRoleModel.Id) @@ -527,6 +530,8 @@ func (impl UserCommonServiceImpl) RemoveRolesAndReturnEliminatedPoliciesForGroup if role, ok := roleIdVsRoleMap[model.RoleId]; ok { isValidAuth := impl.checkRbacForARole(role, token, managerAuth) if !isValidAuth { + impl.logger.Warnw("not authorised to delete role, skipping", "roleId", role.Id, + "entity", role.Entity, "accessType", role.AccessType, "action", role.Action) continue } toBeDeletedRoleGroupRoleMappingsIds = append(toBeDeletedRoleGroupRoleMappingsIds, model.Id) @@ -547,7 +552,9 @@ func (impl UserCommonServiceImpl) RemoveRolesAndReturnEliminatedPoliciesForGroup func (impl UserCommonServiceImpl) checkRbacForARole(role *repository.RoleModel, token string, managerAuth func(resource string, token string, object string) bool) bool { isAuthorised := true switch { - case role.Action == bean2.SUPER_ADMIN || role.AccessType == bean2.APP_ACCESS_TYPE_HELM || role.Entity == bean2.EntityJobs: + case role.Action == bean2.SUPER_ADMIN || role.AccessType == bean2.APP_ACCESS_TYPE_HELM || + role.AccessType == bean2.APP_ACCESS_TYPE_ARGO || role.AccessType == bean2.APP_ACCESS_TYPE_FLUX || + role.Entity == bean2.EntityJobs: isValidAuth := managerAuth(casbin.ResourceGlobal, token, "*") if !isValidAuth { isAuthorised = false @@ -780,6 +787,20 @@ func (impl UserCommonServiceImpl) GetUniqueKeyForAllEntity(entityProcessor Entit default: key = fmt.Sprintf("%s_%s_%s_%s", entityProcessor.GetTeam(), entityProcessor.GetAction(), entityProcessor.GetAccessType(), entityProcessor.GetEntity()) } + } else if entityProcessor.GetEntity() == bean2.ENTITY_APPS && + (entityProcessor.GetAccessType() == bean2.APP_ACCESS_TYPE_ARGO || + entityProcessor.GetAccessType() == bean2.APP_ACCESS_TYPE_FLUX) { + switch baseToConsider { + case bean2.EnvironmentBasedKey: + key = fmt.Sprintf("%s_%s_%s_%s", entityProcessor.GetEntity(), entityProcessor.GetEnvironment(), + entityProcessor.GetAction(), entityProcessor.GetAccessType()) + case bean2.ApplicationBasedKey: + key = fmt.Sprintf("%s_%s_%s_%s", entityProcessor.GetEntity(), entityProcessor.GetEntityName(), + entityProcessor.GetAction(), entityProcessor.GetAccessType()) + default: + key = fmt.Sprintf("%s_%s_%s", entityProcessor.GetEntity(), entityProcessor.GetAction(), + entityProcessor.GetAccessType()) + } } else if len(entityProcessor.GetEntity()) > 0 { if entityProcessor.GetEntity() == bean2.CLUSTER_ENTITIY { key = fmt.Sprintf("%s_%s_%s_%s_%s", entityProcessor.GetEntity(), entityProcessor.GetAction(), entityProcessor.GetCluster(), diff --git a/pkg/auth/user/UserService.go b/pkg/auth/user/UserService.go index 21320edd23..a7724ecc69 100644 --- a/pkg/auth/user/UserService.go +++ b/pkg/auth/user/UserService.go @@ -1480,6 +1480,15 @@ func (impl *UserServiceImpl) BulkDeleteUsers(request *userBean.BulkDeleteRequest // getUserIdsHonoringFilters get the filtered user ids according to the request filters and returns userIds and error(not nil) if any exception is caught. func (impl *UserServiceImpl) getUserIdsHonoringFilters(request *userBean.ListingRequest) ([]int32, error) { + // a filter based bulk delete must always be scoped by at least one concrete filter; an empty/default + // filter must never be allowed to silently widen into "delete every user" (no filter => no filter based bulk delete). + if len(strings.TrimSpace(request.SearchKey)) == 0 && !isStatusFilterApplied(request) { + return nil, &util.ApiError{Code: "400", HttpStatusCode: http.StatusBadRequest, UserMessage: "at least one filter criteria (searchKey or status) is required for a filter based bulk delete"} + } + // applying the same normalization the listing path (GetAllWithFilters) applies before resolving the + // query, so that filter resolution for delete matches filter resolution for listing exactly. + impl.userCommonService.SetDefaultValuesIfNotPresent(request, false) + setStatusFilterType(request) //query to get particular models respecting filters query, queryParams := helper.GetQueryForUserListingWithFilters(request) models, err := impl.userRepository.GetAllExecutingQuery(query, queryParams) diff --git a/pkg/auth/user/UserService_bulkdelete_filter_test.go b/pkg/auth/user/UserService_bulkdelete_filter_test.go new file mode 100644 index 0000000000..85aecd4654 --- /dev/null +++ b/pkg/auth/user/UserService_bulkdelete_filter_test.go @@ -0,0 +1,100 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package user + +import ( + "testing" + + "github.com/devtron-labs/devtron/pkg/auth/user/bean" + "github.com/devtron-labs/devtron/pkg/auth/user/repository" + "github.com/devtron-labs/devtron/pkg/auth/user/repository/helper" + "github.com/stretchr/testify/assert" +) + +// fakeUserRepoForFilterTest is a minimal repository.UserRepository fake that only answers +// GetAllExecutingQuery - the single repository call getUserIdsHonoringFilters makes. Every other +// method is left to panic on the embedded nil interface, since these tests never expect them to run. +type fakeUserRepoForFilterTest struct { + repository.UserRepository + called bool + gotQuery string + gotParams []interface{} + modelsToRet []repository.UserModel +} + +func (f *fakeUserRepoForFilterTest) GetAllExecutingQuery(query string, queryParams []interface{}) ([]repository.UserModel, error) { + f.called = true + f.gotQuery = query + f.gotParams = queryParams + return f.modelsToRet, nil +} + +func newTestUserServiceForFilterTest(repo *fakeUserRepoForFilterTest) *UserServiceImpl { + return &UserServiceImpl{ + userRepository: repo, + userCommonService: &UserCommonServiceImpl{}, + } +} + +// TestGetUserIdsHonoringFilters_RejectsUnscopedFilter is a regression test for the production incident +// where `DELETE /orchestrator/user/bulk` with body {"listingRequest":{"searchKey":"","status":["inactive"]}} +// (an empty search key, intended to be narrowed down by status) deleted every active user on the platform +// instead of only the intended inactive ones. The delete path resolved the filter directly through the +// query builder without the normalization/bound the listing endpoint applies, so an effectively empty +// filter silently widened into "every user". With the fix, a listingRequest carrying no concrete filter +// (status isn't a filterable field in this OSS build) is rejected outright instead of being resolved at all. +func TestGetUserIdsHonoringFilters_RejectsUnscopedFilter(t *testing.T) { + repo := &fakeUserRepoForFilterTest{} + impl := newTestUserServiceForFilterTest(repo) + + // mirrors the exact incident payload once decoded: SearchKey is empty and there is no other + // concrete filter field available on this build's ListingRequest. + request := &bean.ListingRequest{SearchKey: ""} + + ids, err := impl.getUserIdsHonoringFilters(request) + + assert.Error(t, err) + assert.Nil(t, ids) + assert.False(t, repo.called, "no query should ever be executed for an unscoped filter based bulk delete") +} + +// TestGetUserIdsHonoringFilters_MatchesListingQuery asserts that once a concrete filter is supplied, the +// query getUserIdsHonoringFilters resolves is identical - query text and params - to the query the listing +// endpoint (GetAllWithFilters) would build for the exact same input. This is the core of the fix: delete-side +// filter resolution can no longer diverge from listing-side filter resolution (e.g. via a missing default +// page size), so the delete path's resolved id set only ever contains records actually matching the filter. +func TestGetUserIdsHonoringFilters_MatchesListingQuery(t *testing.T) { + matchingUser := repository.UserModel{Id: 42, EmailId: "inactive-user@example.com"} + repo := &fakeUserRepoForFilterTest{modelsToRet: []repository.UserModel{matchingUser}} + impl := newTestUserServiceForFilterTest(repo) + + deleteRequest := &bean.ListingRequest{SearchKey: "inactive-user"} + ids, err := impl.getUserIdsHonoringFilters(deleteRequest) + assert.NoError(t, err) + assert.Equal(t, []int32{42}, ids) + assert.True(t, repo.called) + + // Build the query the listing endpoint would run for the exact same input, applying the exact + // same normalization GetAllWithFilters applies before calling the shared query builder. + listingRequest := &bean.ListingRequest{SearchKey: "inactive-user"} + (&UserCommonServiceImpl{}).SetDefaultValuesIfNotPresent(listingRequest, false) + setStatusFilterType(listingRequest) + listingQuery, listingParams := helper.GetQueryForUserListingWithFilters(listingRequest) + + assert.Equal(t, listingQuery, repo.gotQuery, "delete path query must match listing path query for the same filter") + assert.Equal(t, listingParams, repo.gotParams, "delete path query params must match listing path query params for the same filter") +} diff --git a/pkg/auth/user/UserService_ent.go b/pkg/auth/user/UserService_ent.go index f576f49da3..ab1dc4e6b8 100644 --- a/pkg/auth/user/UserService_ent.go +++ b/pkg/auth/user/UserService_ent.go @@ -119,6 +119,13 @@ func setStatusFilterType(request *userBean.ListingRequest) { return } +// isStatusFilterApplied reports whether the request carries a concrete status filter. +// the OSS ListingRequest has no status filtering concept, so this is always false here; +// an enterprise build that adds status filtering should override this accordingly. +func isStatusFilterApplied(request *userBean.ListingRequest) bool { + return false +} + func setCurrentTimeInUserInfo(request *userBean.ListingRequest) { return } diff --git a/pkg/auth/user/bean/bean.go b/pkg/auth/user/bean/bean.go index 2b03b0ad49..5447775239 100644 --- a/pkg/auth/user/bean/bean.go +++ b/pkg/auth/user/bean/bean.go @@ -66,6 +66,8 @@ const ( const ( DEVTRON_APP = "devtron-app" APP_ACCESS_TYPE_HELM = "helm-app" + APP_ACCESS_TYPE_ARGO = "argo-app" + APP_ACCESS_TYPE_FLUX = "flux-app" EmptyAccessType = "" ) diff --git a/pkg/auth/user/repository/UserAuthRepository.go b/pkg/auth/user/repository/UserAuthRepository.go index ce8fe2940f..7f37ab0b08 100644 --- a/pkg/auth/user/repository/UserAuthRepository.go +++ b/pkg/auth/user/repository/UserAuthRepository.go @@ -22,11 +22,12 @@ package repository import ( "encoding/json" "fmt" + "strings" + "time" + bean3 "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin/bean" "github.com/devtron-labs/devtron/pkg/auth/user/adapter" bean4 "github.com/devtron-labs/devtron/pkg/auth/user/repository/bean" - "strings" - "time" "github.com/devtron-labs/devtron/api/bean" casbin2 "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -73,6 +74,7 @@ type UserAuthRepository interface { GetRoleForClusterEntity(cluster, namespace, group, kind, resource, action string) (RoleModel, error) GetRoleForJobsEntity(entity, team, app, env, act string, workflow string) (RoleModel, error) GetRoleForOtherEntity(team, app, env, act, accessType string, oldValues bool) (RoleModel, error) + GetRoleForExternalGitOpsEntity(app, env, act, accessType string, oldValues bool) (RoleModel, error) GetRoleForChartGroupEntity(entity, app, act, accessType string) (RoleModel, error) } @@ -294,6 +296,13 @@ func (impl UserAuthRepositoryImpl) GetRoleByFilterForAllTypes(roleFieldDto *bean default: { team, app, env, accessType, oldValues := roleFieldDto.Team, roleFieldDto.App, roleFieldDto.Env, roleFieldDto.AccessType, roleFieldDto.OldValues + // External Argo/Flux applications share entity "apps" with devtron-app and + // helm-app but have no project, so they get their own project-free lookup + // instead of being clubbed into the team-keyed queries below. + switch accessType { + case bean2.APP_ACCESS_TYPE_ARGO, bean2.APP_ACCESS_TYPE_FLUX: + return impl.GetRoleForExternalGitOpsEntity(app, env, action, accessType, oldValues) + } return impl.GetRoleForOtherEntity(team, app, env, action, accessType, oldValues) } } @@ -1152,6 +1161,8 @@ func (impl UserAuthRepositoryImpl) GetRoleForOtherEntity(team, app, env, act, ac } else if team == "" && app == "" && env == "" && act == "" { return model, nil } else { + impl.Logger.Warnw("no query branch for the given role filter combination, returning empty role", + "team", team, "app", app, "env", env, "action", act, "accessType", accessType) return model, nil } if err != nil { @@ -1159,3 +1170,40 @@ func (impl UserAuthRepositoryImpl) GetRoleForOtherEntity(team, app, env, act, ac } return model, err } + +func (impl UserAuthRepositoryImpl) GetRoleForExternalGitOpsEntity(app, env, act, accessType string, oldValues bool) (RoleModel, error) { + var model RoleModel + if oldValues { + return model, nil + } + if len(act) == 0 || len(accessType) == 0 { + impl.Logger.Warnw("incomplete filter for external gitops role, returning empty role", + "app", app, "env", env, "action", act, "accessType", accessType) + return model, nil + } + query := "SELECT role.* FROM roles role WHERE coalesce(role.team,'') = ? AND role.action = ? AND role.access_type = ?" + queryParams := []interface{}{EMPTY_PLACEHOLDER_FOR_QUERY, act, accessType} + + if len(app) > 0 { + query += " AND role.entity_name = ?" + queryParams = append(queryParams, app) + } else { + query += " AND coalesce(role.entity_name,'') = ?" + queryParams = append(queryParams, EMPTY_PLACEHOLDER_FOR_QUERY) + } + + if len(env) > 0 { + query += " AND role.environment = ?" + queryParams = append(queryParams, env) + } else { + query += " AND coalesce(role.environment,'') = ?" + queryParams = append(queryParams, EMPTY_PLACEHOLDER_FOR_QUERY) + } + + _, err := impl.dbConnection.Query(&model, query, queryParams...) + if err != nil { + impl.Logger.Errorw("error in getting role for external gitops entity", "err", err, + "app", app, "env", env, "action", act, "accessType", accessType) + } + return model, err +} diff --git a/pkg/fluxApplication/FluxApplicationService.go b/pkg/fluxApplication/FluxApplicationService.go index 779b5b2c63..12a9ec121b 100644 --- a/pkg/fluxApplication/FluxApplicationService.go +++ b/pkg/fluxApplication/FluxApplicationService.go @@ -3,6 +3,9 @@ package fluxApplication import ( "context" "fmt" + "io" + "net/http" + "github.com/devtron-labs/common-lib/utils/k8s/commonBean" "github.com/devtron-labs/devtron/api/connector" "github.com/devtron-labs/devtron/api/helm-app/gRPC" @@ -19,15 +22,15 @@ import ( "github.com/gogo/protobuf/proto" "go.opentelemetry.io/otel" "go.uber.org/zap" - "io" - "net/http" ) type FluxApplicationService interface { - ListFluxApplications(ctx context.Context, clusterIds []int, noStream bool, w http.ResponseWriter) + ListFluxApplications(ctx context.Context, clusterIds []int, noStream bool, w http.ResponseWriter, + token string, fluxAuth func(token string, clusterName string, namespace string, appName string) bool) GetFluxAppDetail(ctx context.Context, app *bean.FluxAppIdentifier) (*bean.FluxApplicationDetailDto, error) HibernateFluxApplication(ctx context.Context, app *bean.FluxAppIdentifier, hibernateRequest *openapi.HibernateRequest) ([]*openapi.HibernateStatus, error) UnHibernateFluxApplication(ctx context.Context, app *bean.FluxAppIdentifier, hibernateRequest *openapi.HibernateRequest) ([]*openapi.HibernateStatus, error) + GetFluxApplicationList(ctx context.Context, clusterIds []int) ([]bean.FluxApplication, error) } type FluxApplicationServiceImpl struct { @@ -92,23 +95,55 @@ func (impl *FluxApplicationServiceImpl) UnHibernateFluxApplication(ctx context.C return response, nil } -func (impl *FluxApplicationServiceImpl) ListFluxApplications(ctx context.Context, clusterIds []int, noStream bool, w http.ResponseWriter) { +func (impl *FluxApplicationServiceImpl) GetFluxApplicationList(ctx context.Context, clusterIds []int) ([]bean.FluxApplication, error) { appStream, err := impl.listApplications(ctx, clusterIds) if err != nil { - impl.logger.Errorw("error in listing flux applications", "clusterIds", clusterIds, "err", err) - return + return nil, err + } + cdPipelineMap, installedAppMap, err := impl.getDevtronManagedMaps(clusterIds) + if err != nil { + return nil, err } + apps := make([]bean.FluxApplication, 0) + for { + appDetail, err := appStream.Recv() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if appDetail.Errored { + impl.logger.Errorw("error in listing flux applications for cluster, skipping it", + "clusterId", appDetail.ClusterId, "errorMsg", appDetail.ErrorMsg) + continue + } + for _, d := range appDetail.FluxApplication { + key := fmt.Sprintf("%v-%s", d.EnvironmentDetail.ClusterId, d.EnvironmentDetail.Namespace) + if _, ok := cdPipelineMap[key][d.Name]; ok { + continue + } + if _, ok := installedAppMap[key][d.Name]; ok { + continue + } + apps = append(apps, toFluxApplication(d)) + } + } + return apps, nil +} + +func (impl *FluxApplicationServiceImpl) getDevtronManagedMaps(clusterIds []int) (map[string]map[string]bool, map[string]map[string]bool, error) { fluxCdPipelines, err := impl.pipelineRepository.GetAppAndEnvDetailsForDeploymentAppTypePipeline(util.PIPELINE_DEPLOYMENT_TYPE_FLUX, clusterIds) if err != nil { impl.logger.Errorw("error in fetching helm app list from DB created using cd_pipelines", "clusters", clusterIds, "err", err) - return + return nil, nil, err } installedHelmApps, err := impl.installedAppRepository.GetAppAndEnvDetailsForDeploymentAppTypeInstalledApps(util.PIPELINE_DEPLOYMENT_TYPE_FLUX, clusterIds) if err != nil { impl.logger.Errorw("error in fetching helm app list from DB created from app store", "clusters", clusterIds, "err", err) - return + return nil, nil, err } cdPipelineMap := make(map[string]map[string]bool) // map of clusterId-namespace, deploymentAppName @@ -129,52 +164,70 @@ func (impl *FluxApplicationServiceImpl) ListFluxApplications(ctx context.Context deploymentAppName := fmt.Sprintf("%s-%s", i.App.AppName, i.Environment.Namespace) installedAppMap[key][deploymentAppName] = true } + + return cdPipelineMap, installedAppMap, nil +} + +func toFluxApplication(app *gRPC.FluxApplication) bean.FluxApplication { + fluxApp := bean.FluxApplication{ + Name: app.Name, + HealthStatus: app.HealthStatus, + SyncStatus: app.SyncStatus, + ClusterId: int(app.EnvironmentDetail.ClusterId), + ClusterName: app.EnvironmentDetail.ClusterName, + Namespace: app.EnvironmentDetail.Namespace, + FluxAppDeploymentType: app.FluxAppDeploymentType, + } + + return fluxApp +} + +func (impl *FluxApplicationServiceImpl) ListFluxApplications(ctx context.Context, clusterIds []int, noStream bool, w http.ResponseWriter, + token string, fluxAuth func(token string, clusterName string, namespace string, appName string) bool) { + if !noStream { + appStream, err := impl.listApplications(ctx, clusterIds) + if err != nil { + impl.logger.Errorw("error in listing flux applications", "clusterIds", clusterIds, "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } + cdPipelineMap, installedAppMap, err := impl.getDevtronManagedMaps(clusterIds) + if err != nil { + impl.logger.Errorw("error in getting devtron managed flux apps", "clusterIds", clusterIds, "err", err) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) + return + } impl.pump.StartStreamWithTransformer(w, func() (proto.Message, error) { return appStream.Recv() }, err, func(message interface{}) interface{} { - return impl.appListRespProtoTransformer(message.(*gRPC.FluxApplicationList), cdPipelineMap, installedAppMap) + return impl.appListRespProtoTransformer(message.(*gRPC.FluxApplicationList), cdPipelineMap, installedAppMap, token, fluxAuth) }) } else { - fluxApps := make([]bean.FluxApplication, 0) - for { - appDetail, err := appStream.Recv() - if err == io.EOF { - break - } - if err != nil { - return + fluxApps, err := impl.GetFluxApplicationList(ctx, clusterIds) + if err != nil { + impl.logger.Errorw("error in getting flux application list", "clusterIds", clusterIds, "err", err) + errored := true + errMsg := err.Error() + appList := bean.FluxAppList{ + Errored: &errored, + ErrorMsg: &errMsg, } - if appDetail.Errored { - appList := bean.FluxAppList{ - Errored: &appDetail.Errored, - ErrorMsg: &appDetail.ErrorMsg, - } - common.WriteJsonResp(w, nil, appList, http.StatusOK) - return - } else { - for _, deployedApp := range appDetail.FluxApplication { - key := fmt.Sprintf("%v-%s", deployedApp.EnvironmentDetail.ClusterId, deployedApp.EnvironmentDetail.Namespace) - if _, ok := cdPipelineMap[key][deployedApp.Name]; ok { - continue - } - if _, ok := installedAppMap[key][deployedApp.Name]; ok { - continue - } - fluxApp := bean.FluxApplication{ - Name: deployedApp.Name, - HealthStatus: deployedApp.HealthStatus, - SyncStatus: deployedApp.SyncStatus, - ClusterId: int(deployedApp.EnvironmentDetail.ClusterId), - ClusterName: deployedApp.EnvironmentDetail.ClusterName, - Namespace: deployedApp.EnvironmentDetail.Namespace, - FluxAppDeploymentType: deployedApp.FluxAppDeploymentType, - } - fluxApps = append(fluxApps, fluxApp) + common.WriteJsonResp(w, nil, appList, http.StatusOK) + return + } + + if fluxAuth != nil { + authorised := make([]bean.FluxApplication, 0, len(fluxApps)) + for _, app := range fluxApps { + if fluxAuth(token, app.ClusterName, app.Namespace, app.Name) { + authorised = append(authorised, app) } } + fluxApps = authorised } + //RBAC enforcer Ends clusterIdsInt32 := sliceUtil.NewSliceFromFuncExec(clusterIds, func(clusterId int) int32 { return int32(clusterId) }) @@ -234,6 +287,11 @@ func (impl *FluxApplicationServiceImpl) listApplications(ctx context.Context, cl } for _, clusterDetail := range clusters { + if clusterDetail.IsVirtualCluster || len(clusterDetail.ErrorInConnecting) != 0 { + impl.logger.Debugw("skipping cluster for flux app listing", "clusterId", clusterDetail.Id, + "isVirtualCluster", clusterDetail.IsVirtualCluster, "errorInConnecting", clusterDetail.ErrorInConnecting) + continue + } config := &gRPC.ClusterConfig{ ApiServerUrl: clusterDetail.ServerUrl, Token: clusterDetail.Config[commonBean.BearerToken], @@ -252,7 +310,8 @@ func (impl *FluxApplicationServiceImpl) listApplications(ctx context.Context, cl return applicationStream, err } -func (impl *FluxApplicationServiceImpl) appListRespProtoTransformer(deployedApps *gRPC.FluxApplicationList, fluxCdPipelines map[string]map[string]bool, fluxInstalledApps map[string]map[string]bool) bean.FluxAppList { +func (impl *FluxApplicationServiceImpl) appListRespProtoTransformer(deployedApps *gRPC.FluxApplicationList, fluxCdPipelines map[string]map[string]bool, fluxInstalledApps map[string]map[string]bool, + token string, fluxAuth func(token string, clusterName string, namespace string, appName string) bool) bean.FluxAppList { appList := bean.FluxAppList{ClusterId: &[]int32{deployedApps.ClusterId}} if deployedApps.Errored { @@ -268,6 +327,11 @@ func (impl *FluxApplicationServiceImpl) appListRespProtoTransformer(deployedApps if _, ok := fluxInstalledApps[key][deployedApp.Name]; ok { continue } + if fluxAuth != nil && !fluxAuth(token, deployedApp.EnvironmentDetail.ClusterName, + deployedApp.EnvironmentDetail.Namespace, deployedApp.Name) { + continue + } + //RBAC enforcer Ends fluxApp := bean.FluxApplication{ Name: deployedApp.Name, HealthStatus: deployedApp.HealthStatus, diff --git a/pkg/k8s/application/k8sApplicationService.go b/pkg/k8s/application/k8sApplicationService.go index 550de045bb..8cbaeb88b9 100644 --- a/pkg/k8s/application/k8sApplicationService.go +++ b/pkg/k8s/application/k8sApplicationService.go @@ -21,6 +21,11 @@ import ( "encoding/json" "errors" "fmt" + "io" + "net/http" + "strconv" + "strings" + "github.com/devtron-labs/common-lib/utils" "github.com/devtron-labs/devtron/api/helm-app/gRPC" client "github.com/devtron-labs/devtron/api/helm-app/service" @@ -33,11 +38,7 @@ import ( "github.com/devtron-labs/devtron/pkg/fluxApplication" bean2 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" bean4 "github.com/devtron-labs/devtron/pkg/k8s/bean" - "io" v1 "k8s.io/client-go/kubernetes/typed/core/v1" - "net/http" - "strconv" - "strings" "github.com/caarlos0/env/v6" k8s2 "github.com/devtron-labs/common-lib/utils/k8s" @@ -369,6 +370,7 @@ func (impl *K8sApplicationServiceImpl) ValidateTerminalRequestQuery(r *http.Requ } resourceRequestBean.ExternalArgoApplicationName = appIdentifier.AppName resourceRequestBean.ClusterId = appIdentifier.ClusterId + resourceRequestBean.ExternalArgoAppIdentifier = appIdentifier request.ClusterId = appIdentifier.ClusterId request.ExternalArgoApplicationName = appIdentifier.AppName request.ExternalArgoApplicationNamespace = appIdentifier.Namespace diff --git a/pkg/pipeline/PipelineStagePipelineConditions_test.go b/pkg/pipeline/PipelineStagePipelineConditions_test.go new file mode 100644 index 0000000000..396175f225 --- /dev/null +++ b/pkg/pipeline/PipelineStagePipelineConditions_test.go @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pipeline + +import ( + "testing" + + "github.com/devtron-labs/devtron/pkg/pipeline/repository" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +type fakePipelineStageRepoForWfRequest struct { + repository.PipelineStageRepository + variables []*repository.PipelineStageStepVariable + conditions []*repository.PipelineStageStepCondition +} + +func (f *fakePipelineStageRepoForWfRequest) GetVariablesByStepId(stepId int) ([]*repository.PipelineStageStepVariable, error) { + return f.variables, nil +} + +func (f *fakePipelineStageRepoForWfRequest) GetConditionsByStepId(stepId int) ([]*repository.PipelineStageStepCondition, error) { + return f.conditions, nil +} + +// Regression guard for the plugin-condition change: the conditions saved on the pipeline step +// (pipeline_stage_step_condition) are the ones that must keep reaching ci-runner. +func TestBuildVariableAndConditionDataForWfRequest_PipelineLevelConditionsAreSent(t *testing.T) { + repo := &fakePipelineStageRepoForWfRequest{ + variables: []*repository.PipelineStageStepVariable{ + {Id: 198, PipelineStageStepId: 225, Name: "FILE_PATH", Format: repository.PIPELINE_STAGE_STEP_VARIABLE_FORMAT_TYPE_STRING, VariableType: repository.PIPELINE_STAGE_STEP_VARIABLE_TYPE_INPUT, ValueType: repository.PIPELINE_STAGE_STEP_VARIABLE_VALUE_TYPE_NEW, Value: "/tmp/other", AllowEmptyValue: true, IsExposed: true, VariableStepIndexInPlugin: 1}, + {Id: 199, PipelineStageStepId: 225, Name: "CONTENT", Format: repository.PIPELINE_STAGE_STEP_VARIABLE_FORMAT_TYPE_STRING, VariableType: repository.PIPELINE_STAGE_STEP_VARIABLE_TYPE_OUTPUT, ValueType: repository.PIPELINE_STAGE_STEP_VARIABLE_VALUE_TYPE_NEW, IsExposed: true, VariableStepIndexInPlugin: 1}, + }, + conditions: []*repository.PipelineStageStepCondition{ + {Id: 15, PipelineStageStepId: 225, ConditionVariableId: 198, ConditionType: repository.PIPELINE_STAGE_STEP_CONDITION_TYPE_TRIGGER, ConditionalOperator: "==", ConditionalValue: "/tmp/other"}, + {Id: 16, PipelineStageStepId: 225, ConditionVariableId: 199, ConditionType: repository.PIPELINE_STAGE_STEP_CONDITION_TYPE_SUCCESS, ConditionalOperator: "==", ConditionalValue: "ok"}, + {Id: 17, PipelineStageStepId: 225, ConditionVariableId: 0, ConditionType: repository.PIPELINE_STAGE_STEP_CONDITION_TYPE_SKIP, ConditionalOperator: "!=", ConditionalValue: "x"}, + }, + } + impl := &PipelineStageServiceImpl{logger: zap.NewNop().Sugar(), pipelineStageRepository: repo} + data, err := impl.buildVariableAndConditionDataForWfRequest(225) + assert.NoError(t, err) + if assert.Len(t, data.GetInputVariables(), 1) { + assert.Equal(t, "/tmp/other", data.GetInputVariables()[0].Value) + assert.Equal(t, 1, data.GetInputVariables()[0].VariableStepIndexInPlugin) + } + assert.Len(t, data.GetOutputVariables(), 1) + triggerSkip := data.GetTriggerSkipConditions() + if assert.Len(t, triggerSkip, 2) { + assert.Equal(t, "FILE_PATH", triggerSkip[0].ConditionOnVariable) + assert.Equal(t, "TRIGGER", triggerSkip[0].ConditionType) + assert.Equal(t, "==", triggerSkip[0].ConditionalOperator) + assert.Equal(t, "/tmp/other", triggerSkip[0].ConditionalValue) + assert.Equal(t, "", triggerSkip[1].ConditionOnVariable, "condition on an unknown variable id is sent without a variable name (pre-existing behaviour)") + } + if assert.Len(t, data.GetSuccessFailureConditions(), 1) { + assert.Equal(t, "CONTENT", data.GetSuccessFailureConditions()[0].ConditionOnVariable) + assert.Equal(t, "PASS", data.GetSuccessFailureConditions()[0].ConditionType) + } +} diff --git a/pkg/pipeline/PipelineStagePluginConditions_test.go b/pkg/pipeline/PipelineStagePluginConditions_test.go new file mode 100644 index 0000000000..5169fd2c51 --- /dev/null +++ b/pkg/pipeline/PipelineStagePluginConditions_test.go @@ -0,0 +1,230 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pipeline + +import ( + "errors" + "testing" + + commonBean "github.com/devtron-labs/common-lib/workflow" + repository2 "github.com/devtron-labs/devtron/pkg/plugin/repository" + "github.com/go-pg/pg" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +// fakePluginRepoForWfRequest serves a plugin definition (steps, script, variables, conditions) to the workflow request +// builders. Any repository method not overridden here panics if called, which is intended. +type fakePluginRepoForWfRequest struct { + repository2.GlobalPluginRepository + steps []*repository2.PluginStep + scripts map[int]*repository2.PluginPipelineScript + variables map[int][]*repository2.PluginStepVariable + variablesErr error + conditions map[int][]*repository2.PluginStepCondition + conditionCalls int +} + +func (f *fakePluginRepoForWfRequest) GetStepsByPluginIds(pluginIds []int) ([]*repository2.PluginStep, error) { + return f.steps, nil +} + +func (f *fakePluginRepoForWfRequest) GetScriptDetailById(id int) (*repository2.PluginPipelineScript, error) { + if script, ok := f.scripts[id]; ok { + return script, nil + } + return nil, pg.ErrNoRows +} + +func (f *fakePluginRepoForWfRequest) GetScriptMappingDetailByScriptId(scriptId int) ([]*repository2.ScriptPathArgPortMapping, error) { + return nil, nil +} + +func (f *fakePluginRepoForWfRequest) GetVariablesByStepId(stepId int) ([]*repository2.PluginStepVariable, error) { + if f.variablesErr != nil { + return nil, f.variablesErr + } + return f.variables[stepId], nil +} + +func (f *fakePluginRepoForWfRequest) GetConditionsByStepId(stepId int) ([]*repository2.PluginStepCondition, error) { + f.conditionCalls++ + return f.conditions[stepId], nil +} + +func newPluginConditionsTestService(repo *fakePluginRepoForWfRequest) *PipelineStageServiceImpl { + return &PipelineStageServiceImpl{logger: zap.NewNop().Sugar(), globalPluginRepository: repo} +} + +func pluginStepVariable(name string, variableType repository2.PluginStepVariableType, exposed bool, value, defaultValue string) *repository2.PluginStepVariable { + return &repository2.PluginStepVariable{ + Id: 1, + Name: name, + Format: repository2.PLUGIN_VARIABLE_FORMAT_TYPE_STRING, + VariableType: variableType, + ValueType: repository2.PLUGIN_VARIABLE_VALUE_TYPE_NEW, + IsExposed: exposed, + Value: value, + DefaultValue: defaultValue, + VariableStepIndex: 1, + } +} + +// pluginWithConditions models a custom plugin saved from the dashboard: one shell step, an exposed input variable +// FILE_PATH with a TRIGGER condition on it and an exposed output variable CONTENT with a PASS condition on it. +func pluginWithConditions() *fakePluginRepoForWfRequest { + input := pluginStepVariable("FILE_PATH", repository2.PLUGIN_VARIABLE_TYPE_INPUT, true, "/tmp/shivam", "") + input.Id = 386 + output := pluginStepVariable("CONTENT", repository2.PLUGIN_VARIABLE_TYPE_OUTPUT, true, "", "") + output.Id = 387 + return &fakePluginRepoForWfRequest{ + steps: []*repository2.PluginStep{ + {Id: 69, PluginId: 71, Name: "Task 1", Index: 1, StepType: repository2.PLUGIN_STEP_TYPE_INLINE, ScriptId: 5}, + }, + scripts: map[int]*repository2.PluginPipelineScript{5: {Type: repository2.SCRIPT_TYPE_SHELL, Script: "echo hi > $FILE_PATH"}}, + variables: map[int][]*repository2.PluginStepVariable{69: {input, output}}, + conditions: map[int][]*repository2.PluginStepCondition{69: { + {Id: 23, PluginStepId: 69, ConditionVariableId: 386, ConditionType: repository2.PLUGIN_CONDITION_TYPE_TRIGGER, ConditionalOperator: "==", ConditionalValue: "/tmp/shivam"}, + {Id: 24, PluginStepId: 69, ConditionVariableId: 387, ConditionType: repository2.PLUGIN_CONDITION_TYPE_SUCCESS, ConditionalOperator: "==", ConditionalValue: "ok"}, + }}, + } +} + +func TestBuildPluginStepDataForWfRequest_PluginConditionsAreNotSentToCiRunner(t *testing.T) { + t.Run("inline plugin step: variables and script are sent, plugin-defined conditions are not", func(t *testing.T) { + repo := pluginWithConditions() + stepData, err := newPluginConditionsTestService(repo).BuildPluginStepDataForWfRequest(repo.steps[0]) + assert.NoError(t, err) + assert.Equal(t, "Task 1", stepData.Name) + assert.Equal(t, 1, stepData.Index) + assert.Equal(t, string(repository2.PLUGIN_STEP_TYPE_INLINE), stepData.StepType) + assert.Equal(t, string(repository2.SCRIPT_TYPE_SHELL), stepData.ExecutorType) + assert.Equal(t, "echo hi > $FILE_PATH", stepData.Script) + if assert.Len(t, stepData.InputVars, 1) { + assert.Equal(t, "FILE_PATH", stepData.InputVars[0].Name) + assert.Equal(t, "/tmp/shivam", stepData.InputVars[0].Value) + } + if assert.Len(t, stepData.OutputVars, 1) { + assert.Equal(t, "CONTENT", stepData.OutputVars[0].Name) + } + assert.Nil(t, stepData.TriggerSkipConditions, "plugin-level trigger/skip conditions must not reach ci-runner") + assert.Nil(t, stepData.SuccessFailureConditions, "plugin-level pass/fail conditions must not reach ci-runner") + assert.Equal(t, 0, repo.conditionCalls, "plugin_step_condition must not even be read for the workflow request") + }) + t.Run("plugin step referencing another plugin: no conditions either", func(t *testing.T) { + repo := pluginWithConditions() + nested := &repository2.PluginStep{Id: 69, PluginId: 71, Name: "nested", Index: 1, StepType: repository2.PLUGIN_STEP_TYPE_REF_PLUGIN, RefPluginId: 42} + stepData, err := newPluginConditionsTestService(repo).BuildPluginStepDataForWfRequest(nested) + assert.NoError(t, err) + assert.Equal(t, "PLUGIN", stepData.ExecutorType) + assert.Equal(t, 42, stepData.RefPluginId) + assert.Len(t, stepData.InputVars, 1) + assert.Nil(t, stepData.TriggerSkipConditions) + assert.Nil(t, stepData.SuccessFailureConditions) + assert.Equal(t, 0, repo.conditionCalls) + }) + t.Run("variable lookup error is propagated", func(t *testing.T) { + repo := pluginWithConditions() + repo.variablesErr = errors.New("db down") + stepData, err := newPluginConditionsTestService(repo).BuildPluginStepDataForWfRequest(repo.steps[0]) + assert.EqualError(t, err, "db down") + assert.Nil(t, stepData) + }) +} + +func TestGetRefPluginStepsByIds_NoPluginConditionOnAnyStep(t *testing.T) { + repo := pluginWithConditions() + pluginIdStepsMap, err := newPluginConditionsTestService(repo).GetRefPluginStepsByIds([]int{71}, map[int]bool{71: true}) + assert.NoError(t, err) + if assert.Len(t, pluginIdStepsMap[71], 1) { + assert.Nil(t, pluginIdStepsMap[71][0].TriggerSkipConditions) + assert.Nil(t, pluginIdStepsMap[71][0].SuccessFailureConditions) + assert.Len(t, pluginIdStepsMap[71][0].InputVars, 1) + } + assert.Equal(t, 0, repo.conditionCalls) +} + +func TestBuildPluginVariableDataForWfRequest(t *testing.T) { + newRepo := func(vars ...*repository2.PluginStepVariable) *fakePluginRepoForWfRequest { + return &fakePluginRepoForWfRequest{variables: map[int][]*repository2.PluginStepVariable{69: vars}} + } + t.Run("exposed input without default uses the stored value, non-exposed input uses its default", func(t *testing.T) { + exposed := pluginStepVariable("A", repository2.PLUGIN_VARIABLE_TYPE_INPUT, true, "user", "") + exposed.VariableStepIndexInPlugin = 3 // only meaningful for plugins nested in plugins; must be passed through untouched + internal := pluginStepVariable("B", repository2.PLUGIN_VARIABLE_TYPE_INPUT, false, "ignored", "default") + exposedWithDefault := pluginStepVariable("C", repository2.PLUGIN_VARIABLE_TYPE_INPUT, true, "", "default") + inputs, outputs, err := newPluginConditionsTestService(newRepo(exposed, internal, exposedWithDefault)).BuildPluginVariableDataForWfRequest(69) + assert.NoError(t, err) + assert.Empty(t, outputs) + if assert.Len(t, inputs, 3) { + assert.Equal(t, "user", inputs[0].Value) + assert.Equal(t, "default", inputs[1].Value) + assert.Equal(t, "", inputs[2].Value, "exposed variable value is filled from the pipeline step by ci-runner") + assert.Equal(t, commonBean.VariableTypeValue, inputs[0].VariableType) + assert.Equal(t, commonBean.FormatTypeString, inputs[0].Format) + assert.Equal(t, 3, inputs[0].VariableStepIndexInPlugin, "nested-plugin step index is passed through") + assert.Equal(t, 0, inputs[1].VariableStepIndexInPlugin) + } + }) + t.Run("value types map to ci-runner variable types", func(t *testing.T) { + global := pluginStepVariable("G", repository2.PLUGIN_VARIABLE_TYPE_INPUT, false, "", "") + global.ValueType = repository2.PLUGIN_VARIABLE_VALUE_TYPE_GLOBAL + global.ReferenceVariableName = "DOCKER_IMAGE" + previous := pluginStepVariable("P", repository2.PLUGIN_VARIABLE_TYPE_INPUT, false, "", "") + previous.ValueType = repository2.PLUGIN_VARIABLE_VALUE_TYPE_PREVIOUS + previous.PreviousStepIndex = 1 + previous.ReferenceVariableName = "OUT" + exposedPrevious := pluginStepVariable("E", repository2.PLUGIN_VARIABLE_TYPE_INPUT, true, "", "") + exposedPrevious.ValueType = repository2.PLUGIN_VARIABLE_VALUE_TYPE_PREVIOUS + inputs, _, err := newPluginConditionsTestService(newRepo(global, previous, exposedPrevious)).BuildPluginVariableDataForWfRequest(69) + assert.NoError(t, err) + if assert.Len(t, inputs, 3) { + assert.Equal(t, commonBean.VariableTypeRefGlobal, inputs[0].VariableType) + assert.Equal(t, "DOCKER_IMAGE", inputs[0].ReferenceVariableName) + assert.Equal(t, commonBean.VariableTypeRefPlugin, inputs[1].VariableType) + assert.Equal(t, 1, inputs[1].ReferenceVariableStepIndex) + assert.Equal(t, commonBean.VariableTypeValue, inputs[2].VariableType, "exposed previous-step variables are resolved at pipeline level") + } + }) + t.Run("output variables are returned separately", func(t *testing.T) { + out := pluginStepVariable("OUT", repository2.PLUGIN_VARIABLE_TYPE_OUTPUT, true, "", "") + inputs, outputs, err := newPluginConditionsTestService(newRepo(out)).BuildPluginVariableDataForWfRequest(69) + assert.NoError(t, err) + assert.Empty(t, inputs) + if assert.Len(t, outputs, 1) { + assert.Equal(t, "OUT", outputs[0].Name) + } + }) + t.Run("no variables (pg.ErrNoRows) yields empty lists", func(t *testing.T) { + repo := &fakePluginRepoForWfRequest{variablesErr: pg.ErrNoRows} + inputs, outputs, err := newPluginConditionsTestService(repo).BuildPluginVariableDataForWfRequest(69) + assert.NoError(t, err) + assert.Empty(t, inputs) + assert.Empty(t, outputs) + }) + t.Run("repository error is propagated", func(t *testing.T) { + repo := &fakePluginRepoForWfRequest{variablesErr: errors.New("db down")} + _, _, err := newPluginConditionsTestService(repo).BuildPluginVariableDataForWfRequest(69) + assert.EqualError(t, err, "db down") + }) + t.Run("invalid variable format is an error", func(t *testing.T) { + bad := pluginStepVariable("X", repository2.PLUGIN_VARIABLE_TYPE_INPUT, true, "", "") + bad.Format = "BOGUS" + _, _, err := newPluginConditionsTestService(newRepo(bad)).BuildPluginVariableDataForWfRequest(69) + assert.Error(t, err) + }) +} diff --git a/pkg/pipeline/PipelineStageService.go b/pkg/pipeline/PipelineStageService.go index a3a2812c1d..c3e0449a37 100644 --- a/pkg/pipeline/PipelineStageService.go +++ b/pkg/pipeline/PipelineStageService.go @@ -2118,34 +2118,38 @@ func (impl *PipelineStageServiceImpl) BuildPluginStepDataForWfRequest(step *repo stepData.ExecutorType = "PLUGIN" //added only to avoid un-marshaling issues at ci-runner side, will not be used stepData.RefPluginId = step.RefPluginId } - inputVars, outputVars, triggerSkipConditions, successFailureConditions, err := impl.BuildPluginVariableAndConditionDataForWfRequest(step.Id) + inputVars, outputVars, err := impl.BuildPluginVariableDataForWfRequest(step.Id) if err != nil { - impl.logger.Errorw("error in getting variable and conditions data for wf request", "err", err, "stepId", step.Id) + impl.logger.Errorw("error in getting variable data for wf request", "err", err, "stepId", step.Id) return nil, err } stepData.InputVars = inputVars stepData.OutputVars = outputVars - stepData.TriggerSkipConditions = triggerSkipConditions - stepData.SuccessFailureConditions = successFailureConditions + // Conditions saved with the plugin definition (plugin_step_condition) are intentionally not sent for the + // plugin's internal steps. They are only defaults that pre-fill the task when the plugin is attached to a + // pipeline; the conditions to evaluate at trigger time are the ones saved on the pipeline step + // (pipeline_stage_step_condition), which are sent with the pre/post stage step referencing this plugin. return stepData, nil } -func (impl *PipelineStageServiceImpl) BuildPluginVariableAndConditionDataForWfRequest(stepId int) ([]*commonBean.VariableObject, []*commonBean.VariableObject, []*bean.ConditionObject, []*bean.ConditionObject, error) { +// BuildPluginVariableDataForWfRequest builds the input and output variables of a plugin's internal step for the +// workflow request. Plugin step conditions (plugin_step_condition) are deliberately not part of the workflow request: +// they are defaults used to pre-fill a task when the plugin is attached to a pipeline, and the effective conditions +// are the ones saved on the pipeline step (pipeline_stage_step_condition). +func (impl *PipelineStageServiceImpl) BuildPluginVariableDataForWfRequest(stepId int) ([]*commonBean.VariableObject, []*commonBean.VariableObject, error) { var inputVariables []*commonBean.VariableObject var outputVariables []*commonBean.VariableObject //getting all variables in the step variables, err := impl.globalPluginRepository.GetVariablesByStepId(stepId) if err != nil && err != pg.ErrNoRows { impl.logger.Errorw("error in getting variables by stepId", "err", err, "stepId", stepId) - return nil, nil, nil, nil, err + return nil, nil, err } - variableNameIdMap := make(map[int]string) for _, variable := range variables { - variableNameIdMap[variable.Id] = variable.Name format, err := commonBean.NewFormat(variable.Format.String()) if err != nil { impl.logger.Errorw("error in creating format object", "err", err, "format", variable.Format) - return nil, nil, nil, nil, err + return nil, nil, err } variableData := &commonBean.VariableObject{ Name: variable.Name, @@ -2181,30 +2185,7 @@ func (impl *PipelineStageServiceImpl) BuildPluginVariableAndConditionDataForWfRe outputVariables = append(outputVariables, variableData) } } - conditions, err := impl.globalPluginRepository.GetConditionsByStepId(stepId) - if err != nil && err != pg.ErrNoRows { - impl.logger.Errorw("error in getting conditions by stepId", "err", err, "stepId", stepId) - return nil, nil, nil, nil, err - } - var triggerSkipConditions []*bean.ConditionObject - var successFailureConditions []*bean.ConditionObject - for _, condition := range conditions { - conditionData := &bean.ConditionObject{ - ConditionalOperator: condition.ConditionalOperator, - ConditionalValue: condition.ConditionalValue, - ConditionType: string(condition.ConditionType), - } - varName, ok := variableNameIdMap[condition.ConditionVariableId] - if ok { - conditionData.ConditionOnVariable = varName - } - if condition.ConditionType == repository2.PLUGIN_CONDITION_TYPE_TRIGGER || condition.ConditionType == repository2.PLUGIN_CONDITION_TYPE_SKIP { - triggerSkipConditions = append(triggerSkipConditions, conditionData) - } else if condition.ConditionType == repository2.PLUGIN_CONDITION_TYPE_SUCCESS || condition.ConditionType == repository2.PLUGIN_CONDITION_TYPE_FAIL { - successFailureConditions = append(successFailureConditions, conditionData) - } - } - return inputVariables, outputVariables, triggerSkipConditions, successFailureConditions, nil + return inputVariables, outputVariables, nil } //BuildPrePostAndRefPluginStepsDataForWfRequest and related methods ends diff --git a/pkg/userResource/UserResourceExtendedService.go b/pkg/userResource/UserResourceExtendedService.go index 358a5000e3..8d78cbb741 100644 --- a/pkg/userResource/UserResourceExtendedService.go +++ b/pkg/userResource/UserResourceExtendedService.go @@ -2,14 +2,18 @@ package userResource import ( "context" + "net/http" + apiBean "github.com/devtron-labs/devtron/api/userResource/bean" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" "github.com/devtron-labs/devtron/pkg/appStore/chartGroup" "github.com/devtron-labs/devtron/pkg/appWorkflow" + argoApplication2 "github.com/devtron-labs/devtron/pkg/argoApplication" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/environment" + "github.com/devtron-labs/devtron/pkg/fluxApplication" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" "github.com/devtron-labs/devtron/pkg/team" "github.com/devtron-labs/devtron/pkg/userResource/adapter" @@ -18,7 +22,6 @@ import ( "github.com/devtron-labs/devtron/util/commonEnforcementFunctionsUtil" "github.com/devtron-labs/devtron/util/rbac" "go.uber.org/zap" - "net/http" ) type UserResourceExtendedServiceImpl struct { @@ -39,13 +42,15 @@ func NewUserResourceExtendedServiceImpl(logger *zap.SugaredLogger, teamService t clusterService cluster.ClusterService, rbacEnforcementUtil commonEnforcementFunctionsUtil.CommonEnforcementUtil, enforcerUtil rbac.EnforcerUtil, - enforcer casbin.Enforcer) *UserResourceExtendedServiceImpl { + enforcer casbin.Enforcer, + argoService argoApplication2.ArgoApplicationService, + fluxService fluxApplication.FluxApplicationService) *UserResourceExtendedServiceImpl { return &UserResourceExtendedServiceImpl{ logger: logger, chartGroupService: chartGroupService, appListingService: appListingService, appWorkflowService: appWorkflowService, - UserResourceServiceImpl: NewUserResourceServiceImpl(logger, teamService, envService, clusterService, k8sApplicationService, enforcerUtil, rbacEnforcementUtil, enforcer, appService), + UserResourceServiceImpl: NewUserResourceServiceImpl(logger, teamService, envService, clusterService, k8sApplicationService, enforcerUtil, rbacEnforcementUtil, enforcer, appService, argoService, fluxService), } } diff --git a/pkg/userResource/UserResourceRbacExtendedService.go b/pkg/userResource/UserResourceRbacExtendedService.go index 0f6c30205d..eb146d5c3e 100644 --- a/pkg/userResource/UserResourceRbacExtendedService.go +++ b/pkg/userResource/UserResourceRbacExtendedService.go @@ -65,6 +65,24 @@ func (impl *UserResourceServiceImpl) enforceRbacForHelmAppsListing(token string, return adapter.BuildUserResourceResponseDto(resourceOptions.TeamAppResp), nil } +func (impl *UserResourceServiceImpl) enforceRbacForArgoAppsListing(token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error) { + isAuthorised := impl.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*") + if !isAuthorised { + impl.logger.Errorw("user is unauthorized enforceRbacForArgoAppsListing") + return adapter.BuildNullDataUserResourceResponseDto(), nil + } + return adapter.BuildUserResourceResponseDto(resourceOptions.ExternalGitOpsAppResp), nil +} + +func (impl *UserResourceServiceImpl) enforceRbacForFluxAppsListing(token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error) { + isAuthorised := impl.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*") + if !isAuthorised { + impl.logger.Errorw("user is unauthorized enforceRbacForFluxAppsListing") + return adapter.BuildNullDataUserResourceResponseDto(), nil + } + return adapter.BuildUserResourceResponseDto(resourceOptions.ExternalGitOpsAppResp), nil +} + func (impl *UserResourceServiceImpl) enforceRbacForJobs(token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error) { isAuthorised := impl.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*") if !isAuthorised { diff --git a/pkg/userResource/UserResourceRbacService.go b/pkg/userResource/UserResourceRbacService.go index 6e0e75e891..2ee5ef8ae8 100644 --- a/pkg/userResource/UserResourceRbacService.go +++ b/pkg/userResource/UserResourceRbacService.go @@ -16,10 +16,13 @@ func (impl *UserResourceServiceImpl) enforceRbacForTeamForHelmApp(token string, return adapter.BuildUserResourceResponseDto(resourceOptions.TeamsResp), nil } -func (impl *UserResourceServiceImpl) enforceRbacForEnvForHelmApp(token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error) { +// enforceRbacForEnvOptions gates the environment options dropdown. It serves all three +// external application types — helm-app, argo-app and flux-app — because the options are +// cluster/namespace pairs regardless of who deployed the application. +func (impl *UserResourceServiceImpl) enforceRbacForEnvOptions(token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error) { isAuthorised := impl.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*") if !isAuthorised { - impl.logger.Errorw("user is unauthorized to enforceRbacForEnvForHelmApp") + impl.logger.Errorw("user is unauthorized to enforceRbacForEnvOptions") return adapter.BuildNullDataUserResourceResponseDto(), nil } return adapter.BuildUserResourceResponseDto(resourceOptions.HelmEnvResp), nil diff --git a/pkg/userResource/UserResourceService.go b/pkg/userResource/UserResourceService.go index 48274c5fe8..49d44e273d 100644 --- a/pkg/userResource/UserResourceService.go +++ b/pkg/userResource/UserResourceService.go @@ -2,23 +2,27 @@ package userResource import ( "context" + "net/http" + apiBean "github.com/devtron-labs/devtron/api/userResource/bean" app2 "github.com/devtron-labs/devtron/internal/sql/repository/app" "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/app" + argoApplication2 "github.com/devtron-labs/devtron/pkg/argoApplication" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/cluster" "github.com/devtron-labs/devtron/pkg/cluster/environment" + "github.com/devtron-labs/devtron/pkg/fluxApplication" application2 "github.com/devtron-labs/devtron/pkg/k8s/application" bean4 "github.com/devtron-labs/devtron/pkg/k8s/bean" "github.com/devtron-labs/devtron/pkg/team" + "github.com/devtron-labs/devtron/pkg/userResource/adapter" bean5 "github.com/devtron-labs/devtron/pkg/userResource/bean" "github.com/devtron-labs/devtron/pkg/userResource/helper" "github.com/devtron-labs/devtron/util/commonEnforcementFunctionsUtil" "github.com/devtron-labs/devtron/util/rbac" "go.uber.org/zap" - "net/http" ) type UserResourceService interface { @@ -26,15 +30,17 @@ type UserResourceService interface { params *apiBean.PathParams) (*bean5.UserResourceResponseDto, error) } type UserResourceServiceImpl struct { - logger *zap.SugaredLogger - teamService team.TeamService - envService environment.EnvironmentService - clusterService cluster.ClusterService - k8sApplicationService application2.K8sApplicationService - enforcerUtil rbac.EnforcerUtil - rbacEnforcementUtil commonEnforcementFunctionsUtil.CommonEnforcementUtil - enforcer casbin.Enforcer - appService app.AppCrudOperationService + logger *zap.SugaredLogger + teamService team.TeamService + envService environment.EnvironmentService + clusterService cluster.ClusterService + k8sApplicationService application2.K8sApplicationService + enforcerUtil rbac.EnforcerUtil + rbacEnforcementUtil commonEnforcementFunctionsUtil.CommonEnforcementUtil + enforcer casbin.Enforcer + appService app.AppCrudOperationService + argoApplicationService argoApplication2.ArgoApplicationService + fluxApplicationService fluxApplication.FluxApplicationService } func NewUserResourceServiceImpl(logger *zap.SugaredLogger, @@ -45,17 +51,21 @@ func NewUserResourceServiceImpl(logger *zap.SugaredLogger, enforcerUtil rbac.EnforcerUtil, rbacEnforcementUtil commonEnforcementFunctionsUtil.CommonEnforcementUtil, enforcer casbin.Enforcer, - appService app.AppCrudOperationService) *UserResourceServiceImpl { + appService app.AppCrudOperationService, + argoApplicationService argoApplication2.ArgoApplicationService, + fluxApplicationService fluxApplication.FluxApplicationService) *UserResourceServiceImpl { return &UserResourceServiceImpl{ - logger: logger, - teamService: teamService, - envService: envService, - clusterService: clusterService, - k8sApplicationService: k8sApplicationService, - enforcerUtil: enforcerUtil, - rbacEnforcementUtil: rbacEnforcementUtil, - enforcer: enforcer, - appService: appService, + logger: logger, + teamService: teamService, + envService: envService, + clusterService: clusterService, + k8sApplicationService: k8sApplicationService, + enforcerUtil: enforcerUtil, + rbacEnforcementUtil: rbacEnforcementUtil, + enforcer: enforcer, + appService: appService, + argoApplicationService: argoApplicationService, + fluxApplicationService: fluxApplicationService, } } @@ -121,7 +131,43 @@ func (impl *UserResourceServiceImpl) getHelmAppResourceOptions(context context.C return bean5.NewResourceOptionsDto().WithTeamAppResp(apps), nil } -func (impl *UserResourceServiceImpl) getHelmEnvResourceOptions(context context.Context, token string, +func (impl *UserResourceServiceImpl) getArgoAppResourceOptions(context context.Context, token string, + reqBean *apiBean.ResourceOptionsReqDto, params *apiBean.PathParams) (*bean5.ResourceOptionsDto, error) { + clusterIds, err := helper.GetValidatedClusterIds(reqBean) + if err != nil { + impl.logger.Errorw("error encountered in getArgoAppResourceOptions", "err", err) + return nil, err + } + apps, err := impl.argoApplicationService.ListApplications(clusterIds) + if err != nil { + impl.logger.Errorw("error encountered in getArgoAppResourceOptions", "err", err) + return nil, err + } + appDtos := helper.FilterExternalGitOpsAppsByEnvIdentifier( + adapter.ArgoAppToExternalGitOpsApp(apps), reqBean.EnvironmentIdentifiers) + + return bean5.NewResourceOptionsDto().WithExternalGitOpsAppResp(appDtos), nil +} + +func (impl *UserResourceServiceImpl) getFluxAppResourceOptions(context context.Context, token string, + reqBean *apiBean.ResourceOptionsReqDto, params *apiBean.PathParams) (*bean5.ResourceOptionsDto, error) { + clusterIds, err := helper.GetValidatedClusterIds(reqBean) + if err != nil { + impl.logger.Errorw("error encountered in getFluxAppResourceOptions", "err", err) + return nil, err + } + apps, err := impl.fluxApplicationService.GetFluxApplicationList(context, clusterIds) + if err != nil { + impl.logger.Errorw("error encountered in getFluxAppResourceOptions", "err", err) + return nil, err + } + appDtos := helper.FilterExternalGitOpsAppsByEnvIdentifier( + adapter.FluxAppToExternalGitOpsApp(apps), reqBean.EnvironmentIdentifiers) + + return bean5.NewResourceOptionsDto().WithExternalGitOpsAppResp(appDtos), nil +} + +func (impl *UserResourceServiceImpl) getCombinedEnvResourceOptions(context context.Context, token string, reqBean *apiBean.ResourceOptionsReqDto, params *apiBean.PathParams) (*bean5.ResourceOptionsDto, error) { // get helm env resource options diff --git a/pkg/userResource/adapter/adapter.go b/pkg/userResource/adapter/adapter.go index 874b61b8f2..73b30b9a3c 100644 --- a/pkg/userResource/adapter/adapter.go +++ b/pkg/userResource/adapter/adapter.go @@ -4,6 +4,8 @@ import ( bean2 "github.com/devtron-labs/devtron/api/userResource/bean" "github.com/devtron-labs/devtron/internal/sql/repository/helper" "github.com/devtron-labs/devtron/pkg/app" + bean3 "github.com/devtron-labs/devtron/pkg/argoApplication/bean" + bean4 "github.com/devtron-labs/devtron/pkg/fluxApplication/bean" "github.com/devtron-labs/devtron/pkg/userResource/bean" ) @@ -26,3 +28,33 @@ func BuildFetchAppListingReqForJobFromDto(reqBean *bean2.ResourceOptionsReqDto) SortOrder: helper.Asc, // default values set } } + +func ArgoAppToExternalGitOpsApp(applications []*bean3.ArgoApplicationListDto) []*bean.ExternalGitOpsAppDto { + result := make([]*bean.ExternalGitOpsAppDto, 0, len(applications)) + for _, application := range applications { + appDto := bean.ExternalGitOpsAppDto{ + AppName: application.Name, + Namespace: application.Namespace, + ClusterId: application.ClusterId, + ClusterName: application.ClusterName, + } + result = append(result, &appDto) + } + + return result +} + +func FluxAppToExternalGitOpsApp(applications []bean4.FluxApplication) []*bean.ExternalGitOpsAppDto { + result := make([]*bean.ExternalGitOpsAppDto, 0, len(applications)) + for _, application := range applications { + appDto := bean.ExternalGitOpsAppDto{ + AppName: application.Name, + Namespace: application.Namespace, + ClusterId: application.ClusterId, + ClusterName: application.ClusterName, + } + result = append(result, &appDto) + } + + return result +} diff --git a/pkg/userResource/bean/bean.go b/pkg/userResource/bean/bean.go index 03f07bd071..7d085bb8dd 100644 --- a/pkg/userResource/bean/bean.go +++ b/pkg/userResource/bean/bean.go @@ -15,17 +15,25 @@ type UserResourceResponseDto struct { Data interface{} `json:"data"` } type ResourceOptionsDto struct { - TeamsResp []bean2.TeamRequest - HelmEnvResp []*bean.ClusterEnvDto - ClusterResp []bean3.ClusterBean - NameSpaces []string - ApiResourcesResp *k8s.GetAllApiResourcesResponse - ClusterResourcesResp *k8s.ClusterResourceListMap - TeamAppResp []*app.TeamAppBean - EnvResp []bean.EnvironmentBean - ChartGroupResp *chartGroup.ChartGroupList - JobsResp []*AppView.JobContainer - AppWfsResp *bean4.WorkflowNamesResponse + TeamsResp []bean2.TeamRequest + HelmEnvResp []*bean.ClusterEnvDto + ClusterResp []bean3.ClusterBean + NameSpaces []string + ApiResourcesResp *k8s.GetAllApiResourcesResponse + ClusterResourcesResp *k8s.ClusterResourceListMap + TeamAppResp []*app.TeamAppBean + EnvResp []bean.EnvironmentBean + ChartGroupResp *chartGroup.ChartGroupList + JobsResp []*AppView.JobContainer + AppWfsResp *bean4.WorkflowNamesResponse + ExternalGitOpsAppResp []*ExternalGitOpsAppDto +} + +type ExternalGitOpsAppDto struct { + AppName string `json:"appName"` + Namespace string `json:"namespace"` + ClusterId int `json:"clusterId"` + ClusterName string `json:"clusterName"` } func NewResourceOptionsDto() *ResourceOptionsDto { @@ -78,6 +86,11 @@ func (r *ResourceOptionsDto) WithAppWfsResp(appWfsResp *bean4.WorkflowNamesRespo return r } +func (r *ResourceOptionsDto) WithExternalGitOpsAppResp(externalGitOpsAppResp []*ExternalGitOpsAppDto) *ResourceOptionsDto { + r.ExternalGitOpsAppResp = externalGitOpsAppResp + return r +} + type Version string type UserResourceKind string @@ -95,6 +108,10 @@ const ( ClusterNamespaces UserResourceKind = "cluster/namespaces" ClusterApiResources UserResourceKind = "cluster/apiResources" ClusterResources UserResourceKind = "cluster/resources" + KindArgoEnvironment UserResourceKind = "environment/argo" + KindFluxEnvironment UserResourceKind = "environment/flux" + KindArgoApplication UserResourceKind = Application + "/argo" + KindFluxApplication UserResourceKind = Application + "/flux" ) const ( diff --git a/pkg/userResource/bean/messages.go b/pkg/userResource/bean/messages.go index b71c8c199e..27571ddf96 100644 --- a/pkg/userResource/bean/messages.go +++ b/pkg/userResource/bean/messages.go @@ -1,8 +1,9 @@ package bean const ( - InvalidPayloadMessage = "Invalid Payload" - InvalidEntityMessage = "Invalid Entity" + InvalidPayloadMessage = "Invalid Payload" + InvalidEntityMessage = "Invalid Entity" + InvalidClusterIdMessage = "Invalid clusterId" ) // messages diff --git a/pkg/userResource/helper/helper.go b/pkg/userResource/helper/helper.go index 58251e3c91..4fe89415b2 100644 --- a/pkg/userResource/helper/helper.go +++ b/pkg/userResource/helper/helper.go @@ -1,10 +1,13 @@ package helper import ( + "fmt" + "net/http" + "strings" + apiBean "github.com/devtron-labs/devtron/api/userResource/bean" "github.com/devtron-labs/devtron/internal/util" bean5 "github.com/devtron-labs/devtron/pkg/userResource/bean" - "net/http" ) func ValidateResourceOptionReqBean(reqBean *apiBean.ResourceOptionsReqDto) error { @@ -16,3 +19,50 @@ func ValidateResourceOptionReqBean(reqBean *apiBean.ResourceOptionsReqDto) error } return nil } + +func GetValidatedClusterIds(reqBean *apiBean.ResourceOptionsReqDto) ([]int, error) { + invalid := util.GetApiErrorAdapter(http.StatusBadRequest, "400", + bean5.InvalidClusterIdMessage, bean5.InvalidClusterIdMessage) + if reqBean == nil { + return nil, invalid + } + seen := make(map[int]bool, len(reqBean.ClusterIds)+1) + clusterIds := make([]int, 0, len(reqBean.ClusterIds)+1) + appendIfValid := func(clusterId int) { + if clusterId > 0 && !seen[clusterId] { + seen[clusterId] = true + clusterIds = append(clusterIds, clusterId) + } + } + for _, clusterId := range reqBean.ClusterIds { + appendIfValid(clusterId) + } + if reqBean.ResourceRequestBean != nil { + appendIfValid(reqBean.ClusterId) + } + if len(clusterIds) == 0 { + return nil, invalid + } + return clusterIds, nil +} + +func FilterExternalGitOpsAppsByEnvIdentifier(apps []*bean5.ExternalGitOpsAppDto, envIdentifiers []string) []*bean5.ExternalGitOpsAppDto { + if len(envIdentifiers) == 0 { + return apps + } + selected := make(map[string]bool, len(envIdentifiers)) + for _, identifier := range envIdentifiers { + selected[strings.ToLower(identifier)] = true + } + filtered := make([]*bean5.ExternalGitOpsAppDto, 0, len(apps)) + for _, app := range apps { + if app == nil { + continue + } + identifier := fmt.Sprintf("%s__%s", app.ClusterName, app.Namespace) + if selected[strings.ToLower(identifier)] { + filtered = append(filtered, app) + } + } + return filtered +} diff --git a/pkg/userResource/logicRouteService.go b/pkg/userResource/logicRouteService.go index 9b9b41ab2c..5447b6db22 100644 --- a/pkg/userResource/logicRouteService.go +++ b/pkg/userResource/logicRouteService.go @@ -3,6 +3,7 @@ package userResource import ( "context" "fmt" + apiBean "github.com/devtron-labs/devtron/api/userResource/bean" bean2 "github.com/devtron-labs/devtron/pkg/auth/user/bean" "github.com/devtron-labs/devtron/pkg/userResource/bean" @@ -17,12 +18,16 @@ func getUserResourceKindWithVersionKey(kind bean.UserResourceKind, version bean. var mapOfUserResourceKindToAllResourceOptionsFunc = map[string]func(impl *UserResourceServiceImpl, context context.Context, token string, reqBean *apiBean.ResourceOptionsReqDto, params *apiBean.PathParams) (*bean.ResourceOptionsDto, error){ getUserResourceKindWithVersionKey(bean.KindTeam, bean.Alpha1Version): (*UserResourceServiceImpl).getTeamResourceOptions, - getUserResourceKindWithVersionKey(bean.KindHelmEnvironment, bean.Alpha1Version): (*UserResourceServiceImpl).getHelmEnvResourceOptions, + getUserResourceKindWithVersionKey(bean.KindHelmEnvironment, bean.Alpha1Version): (*UserResourceServiceImpl).getCombinedEnvResourceOptions, getUserResourceKindWithVersionKey(bean.KindHelmApplication, bean.Alpha1Version): (*UserResourceServiceImpl).getHelmAppResourceOptions, getUserResourceKindWithVersionKey(bean.KindCluster, bean.Alpha1Version): (*UserResourceServiceImpl).getClusterResourceOptions, getUserResourceKindWithVersionKey(bean.ClusterApiResources, bean.Alpha1Version): (*UserResourceServiceImpl).getClusterApiResourceOptions, getUserResourceKindWithVersionKey(bean.ClusterNamespaces, bean.Alpha1Version): (*UserResourceServiceImpl).getClusterNamespacesResourceOptions, getUserResourceKindWithVersionKey(bean.ClusterResources, bean.Alpha1Version): (*UserResourceServiceImpl).getClusterResourceListOptions, + getUserResourceKindWithVersionKey(bean.KindArgoEnvironment, bean.Alpha1Version): (*UserResourceServiceImpl).getCombinedEnvResourceOptions, + getUserResourceKindWithVersionKey(bean.KindFluxEnvironment, bean.Alpha1Version): (*UserResourceServiceImpl).getCombinedEnvResourceOptions, + getUserResourceKindWithVersionKey(bean.KindArgoApplication, bean.Alpha1Version): (*UserResourceServiceImpl).getArgoAppResourceOptions, + getUserResourceKindWithVersionKey(bean.KindFluxApplication, bean.Alpha1Version): (*UserResourceServiceImpl).getFluxAppResourceOptions, } func getAllResourceOptionsFunc(kind bean.UserResourceKind, version bean.Version) func(impl *UserResourceServiceImpl, context context.Context, token string, reqBean *apiBean.ResourceOptionsReqDto, params *apiBean.PathParams) (*bean.ResourceOptionsDto, error) { @@ -34,7 +39,7 @@ func getAllResourceOptionsFunc(kind bean.UserResourceKind, version bean.Version) var mapOfUserResourceKindToAllResourceOptionsExtendedFunc = map[string]func(impl *UserResourceExtendedServiceImpl, context context.Context, token string, reqBean *apiBean.ResourceOptionsReqDto, params *apiBean.PathParams) (*bean.ResourceOptionsDto, error){ getUserResourceKindWithVersionKey(bean.KindTeam, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getTeamResourceOptions, - getUserResourceKindWithVersionKey(bean.KindHelmEnvironment, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getHelmEnvResourceOptions, + getUserResourceKindWithVersionKey(bean.KindHelmEnvironment, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getCombinedEnvResourceOptions, getUserResourceKindWithVersionKey(bean.KindHelmApplication, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getHelmAppResourceOptions, getUserResourceKindWithVersionKey(bean.KindCluster, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getClusterResourceOptions, getUserResourceKindWithVersionKey(bean.ClusterApiResources, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getClusterApiResourceOptions, @@ -45,6 +50,10 @@ var mapOfUserResourceKindToAllResourceOptionsExtendedFunc = map[string]func(impl getUserResourceKindWithVersionKey(bean.KindChartGroup, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getChartGroupResourceOptions, getUserResourceKindWithVersionKey(bean.KindJobs, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getJobsResourceOptions, getUserResourceKindWithVersionKey(bean.KindWorkflow, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getAppWfsResourceOptions, + getUserResourceKindWithVersionKey(bean.KindArgoEnvironment, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getCombinedEnvResourceOptions, + getUserResourceKindWithVersionKey(bean.KindFluxEnvironment, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getCombinedEnvResourceOptions, + getUserResourceKindWithVersionKey(bean.KindArgoApplication, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getArgoAppResourceOptions, + getUserResourceKindWithVersionKey(bean.KindFluxApplication, bean.Alpha1Version): (*UserResourceExtendedServiceImpl).getFluxAppResourceOptions, } func getAllResourceOptionsExtendedFunc(kind bean.UserResourceKind, version bean.Version) func(impl *UserResourceExtendedServiceImpl, context context.Context, token string, reqBean *apiBean.ResourceOptionsReqDto, params *apiBean.PathParams) (*bean.ResourceOptionsDto, error) { @@ -57,11 +66,15 @@ func getAllResourceOptionsExtendedFunc(kind bean.UserResourceKind, version bean. var mapOfKindWithEntityAccessTypeKeyToResourceOptionRbacFunc = map[string]func(impl *UserResourceServiceImpl, token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error){ getUserResourceKindWithEntityAccessKey(bean.KindTeam, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceServiceImpl).enforceRbacForTeamForHelmApp, getUserResourceKindWithEntityAccessKey(bean.KindHelmApplication, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceServiceImpl).enforceRbacForHelmAppsListing, - getUserResourceKindWithEntityAccessKey(bean.KindHelmEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceServiceImpl).enforceRbacForEnvForHelmApp, + getUserResourceKindWithEntityAccessKey(bean.KindHelmEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceServiceImpl).enforceRbacForEnvOptions, getUserResourceKindWithEntityAccessKey(bean.KindCluster, bean.Alpha1Version, bean2.CLUSTER_ENTITIY, bean2.EmptyAccessType): (*UserResourceServiceImpl).enforceRbacForClusterList, getUserResourceKindWithEntityAccessKey(bean.ClusterApiResources, bean.Alpha1Version, bean2.CLUSTER_ENTITIY, bean2.EmptyAccessType): (*UserResourceServiceImpl).enforceRbacForClusterApiResource, getUserResourceKindWithEntityAccessKey(bean.ClusterNamespaces, bean.Alpha1Version, bean2.CLUSTER_ENTITIY, bean2.EmptyAccessType): (*UserResourceServiceImpl).enforceRbacForClusterNamespaces, getUserResourceKindWithEntityAccessKey(bean.ClusterResources, bean.Alpha1Version, bean2.CLUSTER_ENTITIY, bean2.EmptyAccessType): (*UserResourceServiceImpl).enforceRbacForClusterResourceList, + getUserResourceKindWithEntityAccessKey(bean.KindArgoEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_ARGO): (*UserResourceServiceImpl).enforceRbacForEnvOptions, + getUserResourceKindWithEntityAccessKey(bean.KindFluxEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_FLUX): (*UserResourceServiceImpl).enforceRbacForEnvOptions, + getUserResourceKindWithEntityAccessKey(bean.KindArgoApplication, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_ARGO): (*UserResourceServiceImpl).enforceRbacForArgoAppsListing, + getUserResourceKindWithEntityAccessKey(bean.KindFluxApplication, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_FLUX): (*UserResourceServiceImpl).enforceRbacForFluxAppsListing, } func getResourceOptionRbacFunc(kind bean.UserResourceKind, version bean.Version, entity string, accessType string) func(impl *UserResourceServiceImpl, token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error) { @@ -74,7 +87,7 @@ func getResourceOptionRbacFunc(kind bean.UserResourceKind, version bean.Version, var mapOfKindWithEntityAccessTypeKeyToResourceOptionRbacExtendedFunc = map[string]func(impl *UserResourceExtendedServiceImpl, token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error){ getUserResourceKindWithEntityAccessKey(bean.KindTeam, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceExtendedServiceImpl).enforceRbacForTeamForHelmApp, getUserResourceKindWithEntityAccessKey(bean.KindHelmApplication, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceExtendedServiceImpl).enforceRbacForHelmAppsListing, - getUserResourceKindWithEntityAccessKey(bean.KindHelmEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceExtendedServiceImpl).enforceRbacForEnvForHelmApp, + getUserResourceKindWithEntityAccessKey(bean.KindHelmEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_HELM): (*UserResourceExtendedServiceImpl).enforceRbacForEnvOptions, getUserResourceKindWithEntityAccessKey(bean.KindCluster, bean.Alpha1Version, bean2.CLUSTER_ENTITIY, bean2.EmptyAccessType): (*UserResourceExtendedServiceImpl).enforceRbacForClusterList, getUserResourceKindWithEntityAccessKey(bean.ClusterApiResources, bean.Alpha1Version, bean2.CLUSTER_ENTITIY, bean2.EmptyAccessType): (*UserResourceExtendedServiceImpl).enforceRbacForClusterApiResource, getUserResourceKindWithEntityAccessKey(bean.ClusterNamespaces, bean.Alpha1Version, bean2.CLUSTER_ENTITIY, bean2.EmptyAccessType): (*UserResourceExtendedServiceImpl).enforceRbacForClusterNamespaces, @@ -87,6 +100,10 @@ var mapOfKindWithEntityAccessTypeKeyToResourceOptionRbacExtendedFunc = map[strin getUserResourceKindWithEntityAccessKey(bean.KindChartGroup, bean.Alpha1Version, bean2.CHART_GROUP_ENTITY, bean2.EmptyAccessType): (*UserResourceExtendedServiceImpl).enforceRbacForChartGroup, getUserResourceKindWithEntityAccessKey(bean.KindJobs, bean.Alpha1Version, bean2.EntityJobs, bean2.EmptyAccessType): (*UserResourceExtendedServiceImpl).enforceRbacForJobs, getUserResourceKindWithEntityAccessKey(bean.KindWorkflow, bean.Alpha1Version, bean2.EntityJobs, bean2.EmptyAccessType): (*UserResourceExtendedServiceImpl).enforceRbacForJobsWfs, + getUserResourceKindWithEntityAccessKey(bean.KindArgoEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_ARGO): (*UserResourceExtendedServiceImpl).enforceRbacForEnvOptions, + getUserResourceKindWithEntityAccessKey(bean.KindFluxEnvironment, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_FLUX): (*UserResourceExtendedServiceImpl).enforceRbacForEnvOptions, + getUserResourceKindWithEntityAccessKey(bean.KindArgoApplication, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_ARGO): (*UserResourceExtendedServiceImpl).enforceRbacForArgoAppsListing, + getUserResourceKindWithEntityAccessKey(bean.KindFluxApplication, bean.Alpha1Version, bean2.ENTITY_APPS, bean2.APP_ACCESS_TYPE_FLUX): (*UserResourceExtendedServiceImpl).enforceRbacForFluxAppsListing, } func getResourceOptionRbacExtendedFunc(kind bean.UserResourceKind, version bean.Version, entity string, accessType string) func(impl *UserResourceExtendedServiceImpl, token string, params *apiBean.PathParams, resourceOptions *bean.ResourceOptionsDto) (*bean.UserResourceResponseDto, error) { diff --git a/scripts/casbin/12_argo_flux_rbac.down.sql b/scripts/casbin/12_argo_flux_rbac.down.sql new file mode 100644 index 0000000000..59b12b5f86 --- /dev/null +++ b/scripts/casbin/12_argo_flux_rbac.down.sql @@ -0,0 +1,2 @@ +DELETE FROM casbin_rule WHERE v0='role:super-admin___' AND v1='argo-app'; +DELETE FROM casbin_rule WHERE v0='role:super-admin___' AND v1='flux-app'; \ No newline at end of file diff --git a/scripts/casbin/12_argo_flux_rbac.up.sql b/scripts/casbin/12_argo_flux_rbac.up.sql new file mode 100644 index 0000000000..17e4b72057 --- /dev/null +++ b/scripts/casbin/12_argo_flux_rbac.up.sql @@ -0,0 +1,3 @@ +INSERT INTO "public"."casbin_rule" ("p_type","v0","v1","v2","v3","v4","v5") VALUES + ('p','role:super-admin___','argo-app','*','*','allow',''), + ('p','role:super-admin___','flux-app','*','*','allow',''); \ No newline at end of file diff --git a/scripts/sql/36504600_dev_admin_app_lifecycle.down.sql b/scripts/sql/36504600_dev_admin_app_lifecycle.down.sql new file mode 100644 index 0000000000..36b659520e --- /dev/null +++ b/scripts/sql/36504600_dev_admin_app_lifecycle.down.sql @@ -0,0 +1,5 @@ +-- Revert: remove `createApp` / `deleteApp` from the `applications` resource whitelist. +UPDATE rbac_policy_resource_detail +SET allowed_actions = array_remove(array_remove(allowed_actions, 'createApp'), 'deleteApp'), + updated_on = now() +WHERE resource = 'applications'; \ No newline at end of file diff --git a/scripts/sql/36504600_dev_admin_app_lifecycle.up.sql b/scripts/sql/36504600_dev_admin_app_lifecycle.up.sql new file mode 100644 index 0000000000..74064d84d1 --- /dev/null +++ b/scripts/sql/36504600_dev_admin_app_lifecycle.up.sql @@ -0,0 +1,13 @@ +-- Dev-Admin (Release 1): decouple application-entity lifecycle from the overloaded create/delete actions. +-- Adds `createApp` / `deleteApp` to the `applications` resource whitelist so custom roles can grant or +-- withhold application create & delete independently of pipeline/workflow/config operations +-- (which remain on `create`/`delete`). +-- Note: jobs are intentionally NOT touched here (job lifecycle is enforced against the jobs resource). +-- Built-in admin/manager/super-admin keep application lifecycle via their wildcard (`*`) action policies +-- and require no change. +UPDATE rbac_policy_resource_detail +SET allowed_actions = array_cat(allowed_actions, ARRAY ['createApp','deleteApp']), + updated_on = now() +WHERE resource = 'applications' + AND allowed_actions IS NOT NULL + AND NOT ('createApp' = ANY (allowed_actions)); \ No newline at end of file diff --git a/scripts/sql/36804600_argo_flux_rbac.down.sql b/scripts/sql/36804600_argo_flux_rbac.down.sql new file mode 100644 index 0000000000..57d8b04829 --- /dev/null +++ b/scripts/sql/36804600_argo_flux_rbac.down.sql @@ -0,0 +1,2 @@ +DELETE FROM "public"."rbac_policy_data" WHERE entity='apps' AND access_type IN ('argo-app','flux-app'); +DELETE FROM "public"."rbac_role_data" WHERE entity='apps' AND access_type IN ('argo-app','flux-app'); \ No newline at end of file diff --git a/scripts/sql/36804600_argo_flux_rbac.up.sql b/scripts/sql/36804600_argo_flux_rbac.up.sql new file mode 100644 index 0000000000..c7ce5e8a0c --- /dev/null +++ b/scripts/sql/36804600_argo_flux_rbac.up.sql @@ -0,0 +1,111 @@ +INSERT INTO "public"."rbac_policy_data" +("entity","access_type","role","policy_data", + "created_on","created_by","updated_on","updated_by","is_preset_role","deleted") +VALUES + ('apps','argo-app','view','{ + "type": { "value": "p", "indexKeyMap": {} }, + "sub": { "value": "argo-app:view_%_%", + "indexKeyMap": { "14": "Env", "16": "App" } }, + "resActObjSet": [ + { "res": { "value": "argo-app", "indexKeyMap": {} }, + "act": { "value": "get", "indexKeyMap": {} }, + "obj": { "value": "%/%", + "indexKeyMap": { "0": "EnvObj", "2": "AppObj" } } }, + { "res": { "value": "global-environment", "indexKeyMap": {} }, + "act": { "value": "get", "indexKeyMap": {} }, + "obj": { "value": "%", "indexKeyMap": { "0": "EnvObj" } } } + ] +}','now()','1','now()','1',true,false), + + ('apps','argo-app','admin','{ + "type": { "value": "p", "indexKeyMap": {} }, + "sub": { "value": "argo-app:admin_%_%", + "indexKeyMap": { "15": "Env", "17": "App" } }, + "resActObjSet": [ + { "res": { "value": "argo-app", "indexKeyMap": {} }, + "act": { "value": "*", "indexKeyMap": {} }, + "obj": { "value": "%/%", + "indexKeyMap": { "0": "EnvObj", "2": "AppObj" } } }, + { "res": { "value": "global-environment", "indexKeyMap": {} }, + "act": { "value": "get", "indexKeyMap": {} }, + "obj": { "value": "%", "indexKeyMap": { "0": "EnvObj" } } } + ] +}','now()','1','now()','1',true,false), + + ('apps','flux-app','view','{ + "type": { "value": "p", "indexKeyMap": {} }, + "sub": { "value": "flux-app:view_%_%", + "indexKeyMap": { "14": "Env", "16": "App" } }, + "resActObjSet": [ + { "res": { "value": "flux-app", "indexKeyMap": {} }, + "act": { "value": "get", "indexKeyMap": {} }, + "obj": { "value": "%/%", + "indexKeyMap": { "0": "EnvObj", "2": "AppObj" } } }, + { "res": { "value": "global-environment", "indexKeyMap": {} }, + "act": { "value": "get", "indexKeyMap": {} }, + "obj": { "value": "%", "indexKeyMap": { "0": "EnvObj" } } } + ] +}','now()','1','now()','1',true,false), + + ('apps','flux-app','admin','{ + "type": { "value": "p", "indexKeyMap": {} }, + "sub": { "value": "flux-app:admin_%_%", + "indexKeyMap": { "15": "Env", "17": "App" } }, + "resActObjSet": [ + { "res": { "value": "flux-app", "indexKeyMap": {} }, + "act": { "value": "*", "indexKeyMap": {} }, + "obj": { "value": "%/%", + "indexKeyMap": { "0": "EnvObj", "2": "AppObj" } } }, + { "res": { "value": "global-environment", "indexKeyMap": {} }, + "act": { "value": "get", "indexKeyMap": {} }, + "obj": { "value": "%", "indexKeyMap": { "0": "EnvObj" } } } + ] +}','now()','1','now()','1',true,false); + +INSERT INTO "public"."rbac_role_data" +("entity","access_type","role","role_display_name","role_description","role_data", + "created_on","created_by","updated_on","updated_by","is_preset_role","deleted") +VALUES + ('apps','argo-app','view','View only', + 'Can view selected Argo CD application(s) and resource manifests of selected application(s)','{ + "role": { "value": "argo-app:view_%_%", + "indexKeyMap": { "14": "Env", "16": "App" } }, + "entityName": { "value": "%", "indexKeyMap": { "0": "App" } }, + "environment":{ "value": "%", "indexKeyMap": { "0": "Env" } }, + "action": { "value": "view", "indexKeyMap": {} }, + "entity": { "value": "%", "indexKeyMap": { "0": "Entity" } }, + "accessType": { "value": "argo-app", "indexKeyMap": {} } +}','now()','1','now()','1',true,false), + + ('apps','argo-app','admin','Admin', + 'Complete access on selected Argo CD application(s)','{ + "role": { "value": "argo-app:admin_%_%", + "indexKeyMap": { "15": "Env", "17": "App" } }, + "entityName": { "value": "%", "indexKeyMap": { "0": "App" } }, + "environment":{ "value": "%", "indexKeyMap": { "0": "Env" } }, + "action": { "value": "admin", "indexKeyMap": {} }, + "entity": { "value": "%", "indexKeyMap": { "0": "Entity" } }, + "accessType": { "value": "argo-app", "indexKeyMap": {} } +}','now()','1','now()','1',true,false), + + ('apps','flux-app','view','View only', + 'Can view selected Flux CD application(s) and resource manifests of selected application(s)','{ + "role": { "value": "flux-app:view_%_%", + "indexKeyMap": { "14": "Env", "16": "App" } }, + "entityName": { "value": "%", "indexKeyMap": { "0": "App" } }, + "environment":{ "value": "%", "indexKeyMap": { "0": "Env" } }, + "action": { "value": "view", "indexKeyMap": {} }, + "entity": { "value": "%", "indexKeyMap": { "0": "Entity" } }, + "accessType": { "value": "flux-app", "indexKeyMap": {} } +}','now()','1','now()','1',true,false), + + ('apps','flux-app','admin','Admin', + 'Complete access on selected Flux CD application(s)','{ + "role": { "value": "flux-app:admin_%_%", + "indexKeyMap": { "15": "Env", "17": "App" } }, + "entityName": { "value": "%", "indexKeyMap": { "0": "App" } }, + "environment":{ "value": "%", "indexKeyMap": { "0": "Env" } }, + "action": { "value": "admin", "indexKeyMap": {} }, + "entity": { "value": "%", "indexKeyMap": { "0": "Entity" } }, + "accessType": { "value": "flux-app", "indexKeyMap": {} } +}','now()','1','now()','1',true,false); \ No newline at end of file diff --git a/util/rbac/EnforcerUtil.go b/util/rbac/EnforcerUtil.go index 51dcd1aaaf..b619a9d2e0 100644 --- a/util/rbac/EnforcerUtil.go +++ b/util/rbac/EnforcerUtil.go @@ -18,6 +18,7 @@ package rbac import ( "fmt" + bean3 "github.com/devtron-labs/devtron/api/helm-app/service/bean" "github.com/devtron-labs/devtron/pkg/app/dbMigration" repository2 "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" bean2 "github.com/devtron-labs/devtron/pkg/k8s/application/bean" @@ -464,6 +465,8 @@ func (impl EnforcerUtilImpl) GetHelmObject(appId int, envId int) (string, string impl.logger.Errorw("error on fetching data for rbac object", "err", err) return fmt.Sprintf("%s/%s/%s", "", "", ""), "" } + appNameForRbac := application.GetAppNameForRbac() + teamName := getTeamNameForHelmRbac(application) clusterName := env.Cluster.ClusterName namespace := env.Namespace @@ -486,31 +489,26 @@ func (impl EnforcerUtilImpl) GetHelmObject(appId int, envId int) (string, string }*/ if environmentIdentifier2 == "" { - return fmt.Sprintf("%s/%s/%s", application.Team.Name, environmentIdentifier, application.AppName), "" + return fmt.Sprintf("%s/%s/%s", teamName, environmentIdentifier, appNameForRbac), "" } - return fmt.Sprintf("%s/%s/%s", application.Team.Name, environmentIdentifier, application.AppName), - fmt.Sprintf("%s/%s/%s", application.Team.Name, environmentIdentifier2, application.AppName) + return fmt.Sprintf("%s/%s/%s", teamName, environmentIdentifier, appNameForRbac), + fmt.Sprintf("%s/%s/%s", teamName, environmentIdentifier2, appNameForRbac) } func (impl EnforcerUtilImpl) GetHelmObjectByAppNameAndEnvId(appName string, envId int) (string, string) { - application, err := impl.appRepo.FindAppAndProjectByAppName(appName) - if err == pg.ErrMultiRows { - application, err = impl.dbMigration.FixMultipleAppsForInstalledApp(appName) - if err != nil { - impl.logger.Errorw("error on fetching data for rbac object", "appName", appName, "err", err) - return fmt.Sprintf("%s/%s/%s", "", "", ""), "" - } - } + env, err := impl.environmentRepository.FindById(envId) if err != nil { impl.logger.Errorw("error on fetching data for rbac object", "err", err) return fmt.Sprintf("%s/%s/%s", "", "", ""), "" } - env, err := impl.environmentRepository.FindById(envId) + application, err := impl.getAppForHelmRbac(appName, env.ClusterId, env.Namespace) if err != nil { - impl.logger.Errorw("error on fetching data for rbac object", "err", err) + impl.logger.Errorw("error on fetching data for rbac object", "appName", appName, "err", err) return fmt.Sprintf("%s/%s/%s", "", "", ""), "" } + appNameForRbac := application.GetAppNameForRbac() + teamName := getTeamNameForHelmRbac(application) clusterName := env.Cluster.ClusterName namespace := env.Namespace environmentIdentifier := env.EnvironmentIdentifier @@ -524,7 +522,7 @@ func (impl EnforcerUtilImpl) GetHelmObjectByAppNameAndEnvId(appName string, envI } } if environmentIdentifier2 == "" { - return strings.ToLower(fmt.Sprintf("%s/%s/%s", application.Team.Name, environmentIdentifier, application.AppName)), "" + return strings.ToLower(fmt.Sprintf("%s/%s/%s", teamName, environmentIdentifier, appNameForRbac)), "" } //TODO - FIX required for futuristic permission for cluster__* all environment for migrated environment identifier only @@ -532,8 +530,50 @@ func (impl EnforcerUtilImpl) GetHelmObjectByAppNameAndEnvId(appName string, envI if !strings.HasPrefix(env.EnvironmentIdentifier, fmt.Sprintf("%s__", env.Cluster.ClusterName)) { environmentIdentifier = fmt.Sprintf("%s__%s", env.Cluster.ClusterName, env.EnvironmentIdentifier) }*/ - return strings.ToLower(fmt.Sprintf("%s/%s/%s", application.Team.Name, environmentIdentifier, application.AppName)), - strings.ToLower(fmt.Sprintf("%s/%s/%s", application.Team.Name, environmentIdentifier2, application.AppName)) + return strings.ToLower(fmt.Sprintf("%s/%s/%s", teamName, environmentIdentifier, appNameForRbac)), + strings.ToLower(fmt.Sprintf("%s/%s/%s", teamName, environmentIdentifier2, appNameForRbac)) +} + +// getTeamNameForHelmRbac returns the project name to be used while building a helm rbac object. +// An external helm app can be linked to chart store without a project being assigned to it, in which case +// team_id stays 0 and the joined Team row is empty. Permissions for such apps are granted under the +// "unassigned" project, so that literal has to be used instead of an empty project name. +func getTeamNameForHelmRbac(application *app.App) string { + if application.TeamId == 0 { + return repository3.UNASSIGNED_PROJECT + } + return application.Team.Name +} + +// getAppForHelmRbac resolves the app row for a helm rbac object, given a name that may either be the +// app_name stored in db or the display_name shown to the user. +// For an external helm app linked to chart store, app_name is the unique identifier +// (--) while callers here generally only have the release name, so the +// identifier is reconstructed from the environment and tried first. The plain-name lookup is kept as a +// fallback for regular chart store apps, for legacy rows created before the unique identifier existed, and +// for callers that already pass the identifier. +func (impl EnforcerUtilImpl) getAppForHelmRbac(appName string, clusterId int, namespace string) (*app.App, error) { + appIdentifier := &bean3.AppIdentifier{ + ClusterId: clusterId, + Namespace: namespace, + ReleaseName: appName, + } + uniqueAppNameIdentifier := appIdentifier.GetUniqueAppNameIdentifier() + application, err := impl.appRepo.FindAppAndProjectByAppName(uniqueAppNameIdentifier) + if err == pg.ErrMultiRows { + application, err = impl.dbMigration.FixMultipleAppsForInstalledApp(uniqueAppNameIdentifier) + } + if application == nil || err == pg.ErrNoRows { + impl.logger.Debugw("app not found by unique identifier, falling back to app name", "appIdentifier", uniqueAppNameIdentifier, "appName", appName) + application, err = impl.appRepo.FindAppAndProjectByAppName(appName) + if err == pg.ErrMultiRows { + application, err = impl.dbMigration.FixMultipleAppsForInstalledApp(appName) + } + } + if err != nil { + return nil, err + } + return application, nil } func (impl EnforcerUtilImpl) GetHelmObjectByProjectIdAndEnvId(teamId int, envId int) (string, string) { diff --git a/util/rbac/EnforcerUtilGitOps.go b/util/rbac/EnforcerUtilGitOps.go new file mode 100644 index 0000000000..546dcf2fab --- /dev/null +++ b/util/rbac/EnforcerUtilGitOps.go @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "fmt" + + "github.com/devtron-labs/devtron/pkg/cluster/read" + "go.uber.org/zap" +) + +// EnforcerUtilGitOps builds RBAC objects for external Argo CD and Flux CD applications. +// +// The object is two segments — __/ — where the first segment +// is the environment_identifier convention used throughout Devtron. There is no project +// segment: external GitOps applications have no Devtron project, so unlike external Helm apps +// there is no team name and no "unassigned" placeholder to fill. +// +// Both segments are always populated, which matters because matchKeyByPart rejects an empty +// segment on either side unconditionally. +type EnforcerUtilGitOps interface { + // GetExternalGitOpsAppObject returns the RBAC object for an external Argo/Flux application. + // Returns an empty string if the cluster cannot be resolved, which callers must treat as + // a denial rather than as a wildcard. + GetExternalGitOpsAppObject(clusterId int, namespace string, appName string) string + // GetExternalGitOpsAppObjectByClusterName is the same, for callers that already hold the + // cluster name and can avoid the lookup. + GetExternalGitOpsAppObjectByClusterName(clusterName string, namespace string, appName string) string +} + +type EnforcerUtilGitOpsImpl struct { + logger *zap.SugaredLogger + clusterReadService read.ClusterReadService +} + +func NewEnforcerUtilGitOpsImpl(logger *zap.SugaredLogger, + clusterReadService read.ClusterReadService) *EnforcerUtilGitOpsImpl { + return &EnforcerUtilGitOpsImpl{ + logger: logger, + clusterReadService: clusterReadService, + } +} + +func (impl EnforcerUtilGitOpsImpl) GetExternalGitOpsAppObject(clusterId int, namespace string, appName string) string { + cluster, err := impl.clusterReadService.FindById(clusterId) + if err != nil { + impl.logger.Errorw("error on fetching cluster for rbac object", "err", err, "clusterId", clusterId) + return "" + } + return impl.GetExternalGitOpsAppObjectByClusterName(cluster.ClusterName, namespace, appName) +} + +func (impl EnforcerUtilGitOpsImpl) GetExternalGitOpsAppObjectByClusterName(clusterName string, namespace string, appName string) string { + if len(clusterName) == 0 || len(namespace) == 0 || len(appName) == 0 { + impl.logger.Errorw("incomplete identifier for rbac object, denying", + "clusterName", clusterName, "namespace", namespace, "appName", appName) + return "" + } + return fmt.Sprintf("%s__%s/%s", clusterName, namespace, appName) +} diff --git a/util/rbac/EnforcerUtilHelm.go b/util/rbac/EnforcerUtilHelm.go index 268092a6e9..1631db5223 100644 --- a/util/rbac/EnforcerUtilHelm.go +++ b/util/rbac/EnforcerUtilHelm.go @@ -177,7 +177,9 @@ func (impl EnforcerUtilHelmImpl) GetAppRBACNameByInstalledAppId(installedAppVers impl.logger.Errorw("error in fetching installed app version data", "err", err) return fmt.Sprintf("%s/%s/%s", "", "", ""), fmt.Sprintf("%s/%s/%s", "", "", "") } - rbacOne := fmt.Sprintf("%s/%s/%s", InstalledApp.App.Team.Name, InstalledApp.Environment.EnvironmentIdentifier, InstalledApp.App.AppName) + appNameForRbac := InstalledApp.App.GetAppNameForRbac() + teamName := getTeamNameForHelmRbac(&InstalledApp.App) + rbacOne := fmt.Sprintf("%s/%s/%s", teamName, InstalledApp.Environment.EnvironmentIdentifier, appNameForRbac) if InstalledApp.Environment.IsVirtualEnvironment { return rbacOne, "" @@ -186,7 +188,7 @@ func (impl EnforcerUtilHelmImpl) GetAppRBACNameByInstalledAppId(installedAppVers var rbacTwo string if !InstalledApp.Environment.IsVirtualEnvironment { if InstalledApp.Environment.EnvironmentIdentifier != InstalledApp.Environment.Cluster.ClusterName+"__"+InstalledApp.Environment.Namespace { - rbacTwo = fmt.Sprintf("%s/%s/%s", InstalledApp.App.Team.Name, InstalledApp.Environment.Cluster.ClusterName+"__"+InstalledApp.Environment.Namespace, InstalledApp.App.AppName) + rbacTwo = fmt.Sprintf("%s/%s/%s", teamName, InstalledApp.Environment.Cluster.ClusterName+"__"+InstalledApp.Environment.Namespace, appNameForRbac) return rbacOne, rbacTwo } } @@ -205,6 +207,6 @@ func (impl EnforcerUtilHelmImpl) GetAppRBACNameByInstalledAppIdAndTeamId(install impl.logger.Errorw("error in fetching project by teamID", "err", err) return fmt.Sprintf("%s/%s/%s", "", "", "") } - rbac := fmt.Sprintf("%s/%s/%s", project.Name, installedApp.Environment.EnvironmentIdentifier, installedApp.App.AppName) + rbac := fmt.Sprintf("%s/%s/%s", project.Name, installedApp.Environment.EnvironmentIdentifier, installedApp.App.GetAppNameForRbac()) return rbac } diff --git a/util/rbac/EnforcerUtilHelmObject_test.go b/util/rbac/EnforcerUtilHelmObject_test.go new file mode 100644 index 0000000000..b0416f5042 --- /dev/null +++ b/util/rbac/EnforcerUtilHelmObject_test.go @@ -0,0 +1,212 @@ +/* + * Copyright (c) 2020-2024. Devtron Inc. + */ + +package rbac + +import ( + "testing" + + "github.com/devtron-labs/devtron/internal/sql/repository/app" + environmentRepository "github.com/devtron-labs/devtron/pkg/cluster/environment/repository" + clusterRepository "github.com/devtron-labs/devtron/pkg/cluster/repository" + teamRepo "github.com/devtron-labs/devtron/pkg/team/repository" + "github.com/go-pg/pg" + "go.uber.org/zap" +) + +// stubAppRepo overrides only the lookups used while building helm rbac objects; every other method of the +// interface is left nil and will panic if a code path unexpectedly reaches it. +type stubAppRepo struct { + app.AppRepository + byName map[string]*app.App + byId map[int]*app.App +} + +func (s *stubAppRepo) FindAppAndProjectByAppName(appName string) (*app.App, error) { + if a, ok := s.byName[appName]; ok { + return a, nil + } + // mirrors the real repository, which returns an empty (non nil) app alongside ErrNoRows + return &app.App{}, pg.ErrNoRows +} + +func (s *stubAppRepo) FindAppAndProjectByAppId(appId int) (*app.App, error) { + if a, ok := s.byId[appId]; ok { + return a, nil + } + return &app.App{}, pg.ErrNoRows +} + +type stubEnvRepo struct { + environmentRepository.EnvironmentRepository + env *environmentRepository.Environment +} + +func (s *stubEnvRepo) FindById(id int) (*environmentRepository.Environment, error) { + if s.env == nil || s.env.Id != id { + return nil, pg.ErrNoRows + } + return s.env, nil +} + +const ( + testClusterName = "default_cluster" + testNamespace = "devtron-demo" + testEnvId = 1 + testClusterId = 1 +) + +// linkedExternalApp is how an external helm app looks in the app table once it is linked to chart store: +// app_name carries the unique identifier while display_name carries the release name. +func linkedExternalApp(id int, releaseName string, teamId int, teamName string) *app.App { + return &app.App{ + Id: id, + AppName: releaseName + "-" + testNamespace + "-1", + DisplayName: releaseName, + TeamId: teamId, + Team: teamRepo.Team{Name: teamName}, + } +} + +func newTestEnforcerUtil(envIdentifier string) *EnforcerUtilImpl { + nginx := linkedExternalApp(1, "nginx", 2, "myproject") + // linked with no project assigned -> team_id stays 0 and the joined team row is empty + redis := linkedExternalApp(2, "redis", 0, "") + // a regular chart store app, which has no display name at all + grafana := &app.App{Id: 3, AppName: "grafana", TeamId: 2, Team: teamRepo.Team{Name: "myproject"}} + + appRepo := &stubAppRepo{ + byName: map[string]*app.App{ + nginx.AppName: nginx, + redis.AppName: redis, + grafana.AppName: grafana, + }, + byId: map[int]*app.App{1: nginx, 2: redis, 3: grafana}, + } + envRepo := &stubEnvRepo{env: &environmentRepository.Environment{ + Id: testEnvId, + Name: testNamespace, + ClusterId: testClusterId, + Namespace: testNamespace, + EnvironmentIdentifier: envIdentifier, + Cluster: &clusterRepository.Cluster{Id: testClusterId, ClusterName: testClusterName}, + }} + return &EnforcerUtilImpl{logger: zap.NewNop().Sugar(), appRepo: appRepo, environmentRepository: envRepo} +} + +func TestGetHelmObjectByAppNameAndEnvId(t *testing.T) { + envIdentifier := testClusterName + "__" + testNamespace + tests := []struct { + name string + appName string + wantObject string + wantObject2 string + }{ + { + name: "external app linked to chart store is resolved by its display name", + // this is the regression: the listing APIs hand over the release name, but app_name in db holds + // the unique identifier, so the lookup used to miss and the object came back as "//" + appName: "nginx", + wantObject: "myproject/" + envIdentifier + "/nginx", + }, + { + name: "external app linked without a project falls under the unassigned project", + appName: "redis", + wantObject: teamRepo.UNASSIGNED_PROJECT + "/" + envIdentifier + "/redis", + }, + { + name: "regular chart store app keeps working via the plain app name", + appName: "grafana", + wantObject: "myproject/" + envIdentifier + "/grafana", + }, + { + name: "caller passing the unique identifier still gets the display name based object", + // some handlers hand over installedApp.App.AppName directly + appName: "nginx-" + testNamespace + "-1", + wantObject: "myproject/" + envIdentifier + "/nginx", + }, + { + name: "unknown app yields an empty object", + appName: "does-not-exist", + wantObject: "//", + }, + } + impl := newTestEnforcerUtil(envIdentifier) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + object, object2 := impl.GetHelmObjectByAppNameAndEnvId(tt.appName, testEnvId) + if object != tt.wantObject { + t.Errorf("object = %q, want %q", object, tt.wantObject) + } + if object2 != tt.wantObject2 { + t.Errorf("object2 = %q, want %q", object2, tt.wantObject2) + } + }) + } +} + +// When the environment identifier is not clusterName__namespace, both the migrated and the futuristic +// permission objects are returned, and each has to carry the display name. +func TestGetHelmObjectByAppNameAndEnvId_FuturisticPermissionObject(t *testing.T) { + impl := newTestEnforcerUtil("devtron-demo") + object, object2 := impl.GetHelmObjectByAppNameAndEnvId("nginx", testEnvId) + if want := "myproject/devtron-demo/nginx"; object != want { + t.Errorf("object = %q, want %q", object, want) + } + if want := "myproject/" + testClusterName + "__" + testNamespace + "/nginx"; object2 != want { + t.Errorf("object2 = %q, want %q", object2, want) + } +} + +// GetHelmObjectByAppNameAndEnvId lowercases its objects, so a mixed case release name still has to line up +// with the policy that was stored for it. +func TestGetHelmObjectByAppNameAndEnvId_LowerCased(t *testing.T) { + envIdentifier := testClusterName + "__" + testNamespace + mixed := &app.App{ + Id: 4, + AppName: "MyRelease-" + testNamespace + "-1", + DisplayName: "MyRelease", + TeamId: 2, + Team: teamRepo.Team{Name: "MyProject"}, + } + impl := newTestEnforcerUtil(envIdentifier) + impl.appRepo.(*stubAppRepo).byName[mixed.AppName] = mixed + + object, _ := impl.GetHelmObjectByAppNameAndEnvId("MyRelease", testEnvId) + if want := "myproject/" + envIdentifier + "/myrelease"; object != want { + t.Errorf("object = %q, want %q", object, want) + } +} + +func TestGetHelmObject(t *testing.T) { + envIdentifier := testClusterName + "__" + testNamespace + impl := newTestEnforcerUtil(envIdentifier) + + // app id lookup always succeeded, but the object used to be built from app_name (the unique identifier) + object, _ := impl.GetHelmObject(1, testEnvId) + if want := "myproject/" + envIdentifier + "/nginx"; object != want { + t.Errorf("object = %q, want %q", object, want) + } + + object, _ = impl.GetHelmObject(2, testEnvId) + if want := teamRepo.UNASSIGNED_PROJECT + "/" + envIdentifier + "/redis"; object != want { + t.Errorf("object = %q, want %q", object, want) + } + + object, _ = impl.GetHelmObject(3, testEnvId) + if want := "myproject/" + envIdentifier + "/grafana"; object != want { + t.Errorf("object = %q, want %q", object, want) + } +} + +func TestGetAppNameForRbac(t *testing.T) { + external := &app.App{AppName: "nginx-devtron-demo-1", DisplayName: "nginx"} + if got := external.GetAppNameForRbac(); got != "nginx" { + t.Errorf("GetAppNameForRbac() = %q, want %q", got, "nginx") + } + regular := &app.App{AppName: "grafana"} + if got := regular.GetAppNameForRbac(); got != "grafana" { + t.Errorf("GetAppNameForRbac() = %q, want %q", got, "grafana") + } +} diff --git a/wire_gen.go b/wire_gen.go index b27117cd9a..a3bc5f5b69 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -1009,7 +1009,8 @@ func InitializeApp() (*App, error) { coreAppRouterImpl := router.NewCoreAppRouterImpl(coreAppRestHandlerImpl) helmAppRestHandlerImpl := client3.NewHelmAppRestHandlerImpl(sugaredLogger, helmAppServiceImpl, enforcerImpl, clusterServiceImplExtended, enforcerUtilHelmImpl, appStoreDeploymentServiceImpl, installedAppDBServiceImpl, userServiceImpl, attributesServiceImpl, serverEnvConfigServerEnvConfig, fluxApplicationServiceImpl, argoApplicationServiceExtendedImpl) helmAppRouterImpl := client3.NewHelmAppRouterImpl(helmAppRestHandlerImpl) - k8sApplicationRestHandlerImpl := application3.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate, environmentVariables, fluxApplicationServiceImpl, argoApplicationReadServiceImpl) + enforcerUtilGitOpsImpl := rbac.NewEnforcerUtilGitOpsImpl(sugaredLogger, clusterReadServiceImpl) + k8sApplicationRestHandlerImpl := application3.NewK8sApplicationRestHandlerImpl(sugaredLogger, k8sApplicationServiceImpl, pumpImpl, terminalSessionHandlerImpl, enforcerImpl, enforcerUtilHelmImpl, enforcerUtilGitOpsImpl, enforcerUtilImpl, helmAppServiceImpl, userServiceImpl, k8sCommonServiceImpl, validate, environmentVariables, fluxApplicationServiceImpl, argoApplicationReadServiceImpl) k8sApplicationRouterImpl := application3.NewK8sApplicationRouterImpl(k8sApplicationRestHandlerImpl) pProfRestHandlerImpl := restHandler.NewPProfRestHandler(userServiceImpl, enforcerImpl) pProfRouterImpl := router.NewPProfRouter(sugaredLogger, pProfRestHandlerImpl) @@ -1098,18 +1099,18 @@ func InitializeApp() (*App, error) { deploymentConfigurationRouterImpl := configDiff3.NewDeploymentConfigurationRouter(deploymentConfigurationRestHandlerImpl) infraConfigRestHandlerImpl := infraConfig.NewInfraConfigRestHandlerImpl(sugaredLogger, infraConfigServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl, validate) infraConfigRouterImpl := infraConfig.NewInfraProfileRouterImpl(infraConfigRestHandlerImpl) - argoApplicationRestHandlerImpl := argoApplication2.NewArgoApplicationRestHandlerImpl(argoApplicationServiceExtendedImpl, argoApplicationReadServiceImpl, sugaredLogger, enforcerImpl) + argoApplicationRestHandlerImpl := argoApplication2.NewArgoApplicationRestHandlerImpl(argoApplicationServiceExtendedImpl, argoApplicationReadServiceImpl, sugaredLogger, enforcerImpl, enforcerUtilGitOpsImpl) argoApplicationRouterImpl := argoApplication2.NewArgoApplicationRouterImpl(argoApplicationRestHandlerImpl) deploymentHistoryServiceImpl := cdPipeline.NewDeploymentHistoryServiceImpl(sugaredLogger, cdHandlerImpl, imageTaggingReadServiceImpl, imageTaggingServiceImpl, pipelineRepositoryImpl, deployedConfigurationHistoryServiceImpl) apiReqDecoderServiceImpl := devtronResource.NewAPIReqDecoderServiceImpl(sugaredLogger, pipelineRepositoryImpl) historyRestHandlerImpl := devtronResource2.NewHistoryRestHandlerImpl(sugaredLogger, enforcerImpl, deploymentHistoryServiceImpl, apiReqDecoderServiceImpl, enforcerUtilImpl) historyRouterImpl := devtronResource2.NewHistoryRouterImpl(historyRestHandlerImpl) devtronResourceRouterImpl := devtronResource2.NewDevtronResourceRouterImpl(historyRouterImpl) - fluxApplicationRestHandlerImpl := fluxApplication2.NewFluxApplicationRestHandlerImpl(fluxApplicationServiceImpl, sugaredLogger, enforcerImpl) + fluxApplicationRestHandlerImpl := fluxApplication2.NewFluxApplicationRestHandlerImpl(fluxApplicationServiceImpl, sugaredLogger, enforcerImpl, enforcerUtilGitOpsImpl) fluxApplicationRouterImpl := fluxApplication2.NewFluxApplicationRouterImpl(fluxApplicationRestHandlerImpl) scanningResultRestHandlerImpl := resourceScan.NewScanningResultRestHandlerImpl(sugaredLogger, userServiceImpl, imageScanServiceImpl, enforcerImpl, enforcerUtilImpl, validate) scanningResultRouterImpl := resourceScan.NewScanningResultRouterImpl(scanningResultRestHandlerImpl) - userResourceExtendedServiceImpl := userResource.NewUserResourceExtendedServiceImpl(sugaredLogger, teamServiceImpl, environmentServiceImpl, appCrudOperationServiceImpl, chartGroupServiceImpl, appListingServiceImpl, appWorkflowServiceImpl, k8sApplicationServiceImpl, clusterServiceImplExtended, commonEnforcementUtilImpl, enforcerUtilImpl, enforcerImpl) + userResourceExtendedServiceImpl := userResource.NewUserResourceExtendedServiceImpl(sugaredLogger, teamServiceImpl, environmentServiceImpl, appCrudOperationServiceImpl, chartGroupServiceImpl, appListingServiceImpl, appWorkflowServiceImpl, k8sApplicationServiceImpl, clusterServiceImplExtended, commonEnforcementUtilImpl, enforcerUtilImpl, enforcerImpl, argoApplicationServiceExtendedImpl, fluxApplicationServiceImpl) restHandlerImpl := userResource2.NewUserResourceRestHandler(sugaredLogger, userServiceImpl, userResourceExtendedServiceImpl) routerImpl := userResource2.NewUserResourceRouterImpl(restHandlerImpl) appManagementServiceImpl := overview.NewAppManagementServiceImpl(sugaredLogger, appRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciWorkflowRepositoryImpl, cdWorkflowRepositoryImpl, environmentRepositoryImpl, teamRepositoryImpl, workflowStageRepositoryImpl, repositoryImpl) From d413bb15e81dadae8f696f9f20b46550725b4f6e Mon Sep 17 00:00:00 2001 From: bhupesh-devtron Date: Thu, 17 Sep 2026 12:46:52 +0530 Subject: [PATCH 10/12] feat: scoped variable support for git repo and container repo (#7036) --- Wire.go | 6 + .../configure/BuildPipelineRestHandler.go | 28 ++-- .../ScopedVariableRestHandler.go | 44 ++++-- pkg/build/trigger/HandlerService.go | 24 ++- pkg/pipeline/CiCdPipelineOrchestrator.go | 142 ++++++++++++++++-- .../GitMaterialVariableSyncService.go | 62 ++++++++ pkg/resourceQualifiers/constants.go | 11 ++ pkg/resourceQualifiers/constants_test.go | 42 ++++++ pkg/variables/RepoFieldVariableResolver.go | 98 ++++++++++++ .../RepoFieldVariableResolver_test.go | 134 +++++++++++++++++ pkg/variables/VariableEntityMappingService.go | 13 ++ pkg/variables/repository/VariableEntityDTO.go | 3 + .../VariableEntityMappingRepository.go | 18 +++ wire_gen.go | 8 +- 14 files changed, 589 insertions(+), 44 deletions(-) create mode 100644 pkg/pipeline/GitMaterialVariableSyncService.go create mode 100644 pkg/resourceQualifiers/constants_test.go create mode 100644 pkg/variables/RepoFieldVariableResolver.go create mode 100644 pkg/variables/RepoFieldVariableResolver_test.go diff --git a/Wire.go b/Wire.go index 75264c635e..1015ce7788 100644 --- a/Wire.go +++ b/Wire.go @@ -345,6 +345,9 @@ func InitializeApp() (*App, error) { pipeline.NewCiCdPipelineOrchestrator, wire.Bind(new(pipeline.CiCdPipelineOrchestrator), new(*pipeline.CiCdPipelineOrchestratorImpl)), + pipeline.NewGitMaterialVariableSyncServiceImpl, + wire.Bind(new(pipeline.GitMaterialVariableSyncService), new(*pipeline.GitMaterialVariableSyncServiceImpl)), + // scoped variables start variables.NewScopedVariableServiceImpl, wire.Bind(new(variables.ScopedVariableService), new(*variables.ScopedVariableServiceImpl)), @@ -367,6 +370,9 @@ func InitializeApp() (*App, error) { variables.NewScopedVariableCMCSManagerImpl, wire.Bind(new(variables.ScopedVariableCMCSManager), new(*variables.ScopedVariableCMCSManagerImpl)), + variables.NewRepoFieldVariableResolverImpl, + wire.Bind(new(variables.RepoFieldVariableResolver), new(*variables.RepoFieldVariableResolverImpl)), + // end gitOpsConfig.NewDevtronAppGitOpConfigServiceImpl, diff --git a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go index 4d2f0414cb..8cdc04422d 100644 --- a/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go +++ b/api/restHandler/app/pipeline/configure/BuildPipelineRestHandler.go @@ -33,6 +33,7 @@ import ( bean2 "github.com/devtron-labs/devtron/pkg/build/pipeline/bean" eventProcessorBean "github.com/devtron-labs/devtron/pkg/eventProcessor/bean" constants2 "github.com/devtron-labs/devtron/pkg/pipeline/constants" + "github.com/devtron-labs/devtron/pkg/variables" "github.com/devtron-labs/devtron/util/stringsUtil" "golang.org/x/exp/maps" @@ -1363,6 +1364,11 @@ func (handler *PipelineConfigRestHandlerImpl) CreateMaterial(w http.ResponseWrit return } for _, gitMaterial := range createMaterialDto.Material { + // a @{{VAR_NAME}} reference isn't a real URL yet - skip the format check here, + // RepoFieldVariableResolver validates and resolves it before use + if variables.IsFullVariableReference(gitMaterial.Url) { + continue + } validationResult, err := handler.ValidateGitMaterialUrl(gitMaterial.GitProviderId, gitMaterial.Url) if err != nil { handler.Logger.Errorw("service err, CreateMaterial", "err", err, "CreateMaterial", createMaterialDto) @@ -1409,16 +1415,20 @@ func (handler *PipelineConfigRestHandlerImpl) UpdateMaterial(w http.ResponseWrit common.WriteJsonResp(w, err, nil, http.StatusBadRequest) return } - validationResult, err := handler.ValidateGitMaterialUrl(updateMaterialDto.Material.GitProviderId, updateMaterialDto.Material.Url) - if err != nil { - handler.Logger.Errorw("service err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) - return - } else { - if !validationResult { - handler.Logger.Errorw("validation err, UpdateMaterial : invalid git material url", "err", err, "gitMaterialUrl", updateMaterialDto.Material.Url, "UpdateMaterial", updateMaterialDto) - common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + // a @{{VAR_NAME}} reference isn't a real URL yet - skip the format check here, + // RepoFieldVariableResolver validates and resolves it before use + if !variables.IsFullVariableReference(updateMaterialDto.Material.Url) { + validationResult, err := handler.ValidateGitMaterialUrl(updateMaterialDto.Material.GitProviderId, updateMaterialDto.Material.Url) + if err != nil { + handler.Logger.Errorw("service err, UpdateMaterial", "err", err, "UpdateMaterial", updateMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusInternalServerError) return + } else { + if !validationResult { + handler.Logger.Errorw("validation err, UpdateMaterial : invalid git material url", "err", err, "gitMaterialUrl", updateMaterialDto.Material.Url, "UpdateMaterial", updateMaterialDto) + common.WriteJsonResp(w, err, nil, http.StatusBadRequest) + return + } } } resourceObject := handler.enforcerUtil.GetAppRBACNameByAppId(updateMaterialDto.AppId) diff --git a/api/restHandler/scopedVariable/ScopedVariableRestHandler.go b/api/restHandler/scopedVariable/ScopedVariableRestHandler.go index 986cefaa0c..ffbc3f97f8 100644 --- a/api/restHandler/scopedVariable/ScopedVariableRestHandler.go +++ b/api/restHandler/scopedVariable/ScopedVariableRestHandler.go @@ -23,6 +23,7 @@ import ( "net/http" "strconv" + "github.com/devtron-labs/common-lib/async" "github.com/devtron-labs/devtron/api/restHandler/common" util2 "github.com/devtron-labs/devtron/internal/util" "github.com/devtron-labs/devtron/pkg/auth/authorisation/casbin" @@ -47,28 +48,32 @@ type ScopedVariableRestHandler interface { } type ScopedVariableRestHandlerImpl struct { - logger *zap.SugaredLogger - userAuthService user.UserService - validator *validator.Validate - pipelineBuilder pipeline.PipelineBuilder - enforcerUtil rbac.EnforcerUtil - enforcer casbin.Enforcer - scopedVariableService variables.ScopedVariableService + logger *zap.SugaredLogger + userAuthService user.UserService + validator *validator.Validate + pipelineBuilder pipeline.PipelineBuilder + enforcerUtil rbac.EnforcerUtil + enforcer casbin.Enforcer + scopedVariableService variables.ScopedVariableService + gitMaterialVariableSyncService pipeline.GitMaterialVariableSyncService + asyncRunnable *async.Runnable } type JsonResponse struct { Manifest *models.ScopedVariableManifest `json:"manifest"` JsonSchema string `json:"jsonSchema"` } -func NewScopedVariableRestHandlerImpl(logger *zap.SugaredLogger, userAuthService user.UserService, validator *validator.Validate, pipelineBuilder pipeline.PipelineBuilder, enforcerUtil rbac.EnforcerUtil, enforcer casbin.Enforcer, scopedVariableService variables.ScopedVariableService) *ScopedVariableRestHandlerImpl { +func NewScopedVariableRestHandlerImpl(logger *zap.SugaredLogger, userAuthService user.UserService, validator *validator.Validate, pipelineBuilder pipeline.PipelineBuilder, enforcerUtil rbac.EnforcerUtil, enforcer casbin.Enforcer, scopedVariableService variables.ScopedVariableService, gitMaterialVariableSyncService pipeline.GitMaterialVariableSyncService, asyncRunnable *async.Runnable) *ScopedVariableRestHandlerImpl { return &ScopedVariableRestHandlerImpl{ - logger: logger, - userAuthService: userAuthService, - validator: validator, - pipelineBuilder: pipelineBuilder, - enforcerUtil: enforcerUtil, - enforcer: enforcer, - scopedVariableService: scopedVariableService, + logger: logger, + userAuthService: userAuthService, + validator: validator, + pipelineBuilder: pipelineBuilder, + enforcerUtil: enforcerUtil, + enforcer: enforcer, + scopedVariableService: scopedVariableService, + gitMaterialVariableSyncService: gitMaterialVariableSyncService, + asyncRunnable: asyncRunnable, } } func (handler *ScopedVariableRestHandlerImpl) CreateVariables(w http.ResponseWriter, r *http.Request) { @@ -132,6 +137,15 @@ func (handler *ScopedVariableRestHandlerImpl) CreateVariables(w http.ResponseWri } return } + // CreateVariables replaces the entire variable set, so any Git Material referencing a + // scoped variable may need its git-sensor copy refreshed - run off-thread since each + // push to git-sensor can take up to its own timeout ceiling, and this must not block + // the response for a save that already succeeded above. + handler.asyncRunnable.Execute(func() { + if syncErr := handler.gitMaterialVariableSyncService.ResyncGitMaterialsForVariableChange(); syncErr != nil { + handler.logger.Errorw("error re-syncing git materials after variable change", "err", syncErr) + } + }) common.WriteJsonResp(w, nil, nil, http.StatusOK) } diff --git a/pkg/build/trigger/HandlerService.go b/pkg/build/trigger/HandlerService.go index a4b97ce611..ff015e8316 100644 --- a/pkg/build/trigger/HandlerService.go +++ b/pkg/build/trigger/HandlerService.go @@ -110,6 +110,7 @@ type HandlerServiceImpl struct { customTagService pipeline2.CustomTagService config *types.CiConfig scopedVariableManager variables.ScopedVariableManager + repoFieldVariableResolver variables.RepoFieldVariableResolver ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator buildxGlobalFlags *BuildxGlobalFlags attributeService attributes.AttributesService @@ -138,6 +139,7 @@ func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.W envRepository repository6.EnvironmentRepository, appRepository appRepository.AppRepository, scopedVariableManager variables.ScopedVariableManager, + repoFieldVariableResolver variables.RepoFieldVariableResolver, customTagService pipeline2.CustomTagService, ciCdPipelineOrchestrator pipeline2.CiCdPipelineOrchestrator, attributeService attributes.AttributesService, pluginInputVariableParser pipeline2.PluginInputVariableParser, @@ -171,6 +173,7 @@ func NewHandlerServiceImpl(Logger *zap.SugaredLogger, workflowService executor.W envRepository: envRepository, appRepository: appRepository, scopedVariableManager: scopedVariableManager, + repoFieldVariableResolver: repoFieldVariableResolver, customTagService: customTagService, ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, buildxGlobalFlags: buildxCacheFlags, @@ -847,7 +850,7 @@ func (impl *HandlerServiceImpl) StartCiWorkflowAndPrepareWfRequest(trigger *type trigger.RuntimeParameters = trigger.RuntimeParameters.AddSystemVariable(k, v) } - workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob) + workflowRequest, err := impl.buildWfRequestForCiPipeline(pipeline, trigger, ciMaterials, savedCiWf, ciWorkflowConfigNamespace, ciPipelineScripts, preCiSteps, postCiSteps, refPluginsData, isJob, scope) if err != nil { impl.Logger.Errorw("make workflow req", "err", err) return nil, nil, nil, err @@ -1006,7 +1009,7 @@ func (impl *HandlerServiceImpl) buildDefaultArtifactLocation(savedWf *pipelineCo return ArtifactLocation } -func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger *types.CiTriggerRequest, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool) (*types.WorkflowRequest, error) { +func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineConfig.CiPipeline, trigger *types.CiTriggerRequest, ciMaterials []*pipelineConfig.CiPipelineMaterial, savedWf *pipelineConfig.CiWorkflow, ciWorkflowConfigNamespace string, ciPipelineScripts []*pipelineConfig.CiPipelineScript, preCiSteps []*pipelineConfigBean.StepObject, postCiSteps []*pipelineConfigBean.StepObject, refPluginsData []*pipelineConfigBean.RefPluginObject, isJob bool, scope resourceQualifiers.Scope) (*types.WorkflowRequest, error) { var ciProjectDetails []pipelineConfigBean.CiProjectDetails commitHashes := trigger.CommitHashes for _, ciMaterial := range ciMaterials { @@ -1015,8 +1018,15 @@ func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineCo continue } commitHashForPipelineId := commitHashes[ciMaterial.Id] + // git_material.url may hold a @{{VAR_NAME}} reference (never resolved into the DB, see + // RepoFieldVariableResolver) - resolve it fresh here rather than at Git Material save time + gitRepository, _, err := impl.repoFieldVariableResolver.ResolveRepoFieldValue(scope, ciMaterial.GitMaterial.Url) + if err != nil { + impl.Logger.Errorw("error resolving git material url", "gitMaterialId", ciMaterial.GitMaterial.Id, "err", err) + return nil, err + } ciProjectDetail := pipelineConfigBean.CiProjectDetails{ - GitRepository: ciMaterial.GitMaterial.Url, + GitRepository: gitRepository, MaterialName: ciMaterial.GitMaterial.Name, CheckoutPath: ciMaterial.GitMaterial.CheckoutPath, FetchSubmodules: ciMaterial.GitMaterial.FetchSubmodules, @@ -1040,7 +1050,6 @@ func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineCo CaCert: ciMaterial.GitMaterial.GitProvider.CaCert, }, } - var err error ciProjectDetail, err = impl.updateCIProjectDetailWithCloningMode(pipeline.AppId, ciMaterial, ciProjectDetail) if err != nil { impl.Logger.Errorw("error, updateCIProjectDetailWithCloningMode", "pipelineId", pipeline.Id, "err", err) @@ -1147,6 +1156,13 @@ func (impl *HandlerServiceImpl) buildWfRequestForCiPipeline(pipeline *pipelineCo } } + // ci_template(_override).docker_repository may hold a @{{VAR_NAME}} reference (never + // resolved at save time, so a variable value update takes effect on the next build) + dockerRepository, _, err = impl.repoFieldVariableResolver.ResolveRepoFieldValue(scope, dockerRepository) + if err != nil { + impl.Logger.Errorw("error resolving container repository", "pipelineId", pipeline.Id, "err", err) + return nil, err + } if checkoutPath == "" { checkoutPath = "./" } diff --git a/pkg/pipeline/CiCdPipelineOrchestrator.go b/pkg/pipeline/CiCdPipelineOrchestrator.go index 82222aaffd..2c757bc348 100644 --- a/pkg/pipeline/CiCdPipelineOrchestrator.go +++ b/pkg/pipeline/CiCdPipelineOrchestrator.go @@ -40,6 +40,9 @@ import ( "github.com/devtron-labs/devtron/pkg/deployment/gitOps/config" util4 "github.com/devtron-labs/devtron/pkg/pipeline/util" "github.com/devtron-labs/devtron/pkg/plugin" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/variables" + variableEntity "github.com/devtron-labs/devtron/pkg/variables/repository" "github.com/devtron-labs/devtron/util/beHelper" "github.com/devtron-labs/devtron/util/sliceUtil" "golang.org/x/exp/slices" @@ -92,6 +95,9 @@ type CiCdPipelineOrchestrator interface { DeleteApp(appId int, userId int32) error CreateMaterials(createMaterialRequest *bean.CreateMaterialDTO) (*bean.CreateMaterialDTO, error) UpdateMaterial(updateMaterialRequest *bean.UpdateMaterialDTO) (*bean.UpdateMaterialDTO, error) + // ResyncGitMaterialToSensor re-resolves the material's @{{VAR_NAME}} URL (if any) and + // pushes the refreshed value to git-sensor - used after a scoped variable's value changes. + ResyncGitMaterialToSensor(materialId int) error CreateCiConf(createRequest *bean.CiConfigRequest, templateId int) (*bean.CiConfigRequest, error) CreateCDPipelines(pipelineRequest *bean.CDPipelineConfigObject, appId int, userId int32, tx *pg.Tx, appName string) (pipelineId int, err error) UpdateCDPipeline(pipelineRequest *bean.CDPipelineConfigObject, userId int32, tx *pg.Tx) (pipeline *pipelineConfig.Pipeline, err error) @@ -155,6 +161,8 @@ type CiCdPipelineOrchestratorImpl struct { workflowCacheConfig types.WorkflowCacheConfig chartReadService read2.ChartReadService globalEnvVariables *util2.GlobalEnvVariables + repoFieldVariableResolver variables.RepoFieldVariableResolver + variableEntityMappingService variables.VariableEntityMappingService } func NewCiCdPipelineOrchestrator( @@ -187,7 +195,9 @@ func NewCiCdPipelineOrchestrator( deploymentConfigService common.DeploymentConfigService, deploymentConfigReadService read.DeploymentConfigReadService, chartReadService read2.ChartReadService, - envVariables *util2.EnvironmentVariables) *CiCdPipelineOrchestratorImpl { + envVariables *util2.EnvironmentVariables, + repoFieldVariableResolver variables.RepoFieldVariableResolver, + variableEntityMappingService variables.VariableEntityMappingService) *CiCdPipelineOrchestratorImpl { _, workflowCacheConfig, err := types.GetCiConfigWithWorkflowCacheConfig() if err != nil { logger.Errorw("Error in getting workflow cache config, continuing with default values", "err", err) @@ -226,6 +236,8 @@ func NewCiCdPipelineOrchestrator( workflowCacheConfig: workflowCacheConfig, chartReadService: chartReadService, globalEnvVariables: envVariables.GlobalEnvVariables, + repoFieldVariableResolver: repoFieldVariableResolver, + variableEntityMappingService: variableEntityMappingService, } } @@ -615,9 +627,14 @@ func (impl CiCdPipelineOrchestratorImpl) PatchMaterialValue(createRequest *bean. } savedTemplateOverride := savedTemplateOverrideBean.CiTemplateOverride - err = impl.createDockerRepoIfNeeded(createRequest.DockerConfigOverride.DockerRegistry, createRequest.DockerConfigOverride.DockerRepository) + resolvedDockerRepository, err := impl.resolveDockerRepositoryForEcr(resourceQualifiers.Scope{AppId: createRequest.AppId}, createRequest.DockerConfigOverride.DockerRepository) if err != nil { - impl.logger.Errorw("error, createDockerRepoIfNeeded", "err", err, "dockerRegistryId", createRequest.DockerConfigOverride.DockerRegistry, "dockerRegistry", createRequest.DockerConfigOverride.DockerRepository) + impl.logger.Errorw("error resolving docker repository", "err", err, "dockerRepository", createRequest.DockerConfigOverride.DockerRepository) + return nil, err + } + err = impl.createDockerRepoIfNeeded(createRequest.DockerConfigOverride.DockerRegistry, resolvedDockerRepository) + if err != nil { + impl.logger.Errorw("error, createDockerRepoIfNeeded", "err", err, "dockerRegistryId", createRequest.DockerConfigOverride.DockerRegistry, "dockerRegistry", resolvedDockerRepository) return nil, err } if savedTemplateOverride != nil && savedTemplateOverride.Id > 0 { @@ -920,6 +937,7 @@ func (impl CiCdPipelineOrchestratorImpl) deleteCiEnvMapping(tx *pg.Tx, ciPipelin return nil } func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConfigRequest, templateId int) (*bean.CiConfigRequest, error) { + repoFieldScope := resourceQualifiers.Scope{AppId: createRequest.AppId} //save pipeline in db start for _, ciPipeline := range createRequest.CiPipelines { argByte, err := json.Marshal(ciPipeline.DockerArgs) @@ -1121,10 +1139,16 @@ func (impl CiCdPipelineOrchestratorImpl) CreateCiConf(createRequest *bean.CiConf UserId: createRequest.UserId, } if !ciPipeline.IsExternal { //pipeline is not [linked, webhook] and overridden, then create template override - err = impl.createDockerRepoIfNeeded(ciPipeline.DockerConfigOverride.DockerRegistry, ciPipeline.DockerConfigOverride.DockerRepository) + resolvedDockerRepository, resolveErr := impl.resolveDockerRepositoryForEcr(repoFieldScope, ciPipeline.DockerConfigOverride.DockerRepository) // silently ignoring the error, since we don't want to fail the pipeline creation - if err != nil { - impl.logger.Warnw("error, createDockerRepoIfNeeded", "err", err, "dockerRegistryId", ciPipeline.DockerConfigOverride.DockerRegistry, "dockerRegistry", ciPipeline.DockerConfigOverride.DockerRepository) + if resolveErr != nil { + impl.logger.Warnw("error resolving docker repository", "err", resolveErr, "dockerRepository", ciPipeline.DockerConfigOverride.DockerRepository) + } else { + err = impl.createDockerRepoIfNeeded(ciPipeline.DockerConfigOverride.DockerRegistry, resolvedDockerRepository) + // silently ignoring the error, since we don't want to fail the pipeline creation + if err != nil { + impl.logger.Warnw("error, createDockerRepoIfNeeded", "err", err, "dockerRegistryId", ciPipeline.DockerConfigOverride.DockerRegistry, "dockerRegistry", resolvedDockerRepository) + } } err := impl.ciTemplateService.Save(ciTemplateBean) if err != nil { @@ -1365,7 +1389,8 @@ func (impl CiCdPipelineOrchestratorImpl) DeleteApp(appId int, userId int32) erro impl.logger.Debug("deleting materials in git_sensor") for _, m := range materials { - err = impl.updateRepositoryToGitSensor(m, "", false) + // m.Active is already false above, so updateRepositoryToGitSensor skips URL resolution - scope is unused here + err = impl.updateRepositoryToGitSensor(m, "", false, resourceQualifiers.Scope{}) if err != nil { impl.logger.Errorw("error in updating to git-sensor", "err", err) return err @@ -1457,6 +1482,15 @@ func (impl CiCdPipelineOrchestratorImpl) CreateMaterials(createMaterialRequest * } materials = append(materials, inputMaterial) } + repoFieldScope := resourceQualifiers.Scope{AppId: createMaterialRequest.AppId} + // validate every @{{VAR_NAME}} reference resolves before committing, so a typo'd/undefined + // variable can't leave a Git Material row with no matching git-sensor entry + for _, material := range materials { + if _, err = impl.resolveGitMaterialUrl(repoFieldScope, material.Url); err != nil { + impl.logger.Errorw("error validating git material url", "err", err) + return nil, err + } + } // moving transaction before addRepositoryToGitSensor as commiting transaction after addRepositoryToGitSensor was causing problems // in case request was cancelled data was getting saved in git sensor but not getting saved in orchestrator db. Same is done in update flow. err = impl.transactionManager.CommitTx(tx) @@ -1464,7 +1498,7 @@ func (impl CiCdPipelineOrchestratorImpl) CreateMaterials(createMaterialRequest * impl.logger.Errorw("error in committing tx Create material", "err", err, "materials", materials) return nil, err } - err = impl.addRepositoryToGitSensor(materials, "") + err = impl.addRepositoryToGitSensor(materials, "", repoFieldScope) if err != nil { impl.logger.Errorw("error in updating to sensor", "err", err) return nil, err @@ -1484,6 +1518,15 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean. impl.logger.Errorw("err", "err", err) return nil, err } + repoFieldScope := resourceQualifiers.Scope{AppId: updateMaterialDTO.AppId} + // validate the @{{VAR_NAME}} reference resolves before committing, so a typo'd/undefined + // variable can't leave a Git Material row with no matching git-sensor entry + if updatedMaterial.Active { + if _, err = impl.resolveGitMaterialUrl(repoFieldScope, updatedMaterial.Url); err != nil { + impl.logger.Errorw("error validating git material url", "err", err) + return nil, err + } + } err = impl.transactionManager.CommitTx(tx) if err != nil { impl.logger.Errorw("error in committing tx Create material", "err", err) @@ -1491,7 +1534,7 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean. } err = impl.updateRepositoryToGitSensor(updatedMaterial, "", - updateMaterialDTO.Material.CreateBackup) + updateMaterialDTO.Material.CreateBackup, repoFieldScope) if err != nil { impl.logger.Errorw("error in updating to git-sensor", "err", err) return nil, err @@ -1499,11 +1542,71 @@ func (impl CiCdPipelineOrchestratorImpl) UpdateMaterial(updateMaterialDTO *bean. return updateMaterialDTO, nil } +// resolveDockerRepositoryForEcr resolves a possible @{{VAR_NAME}} reference in the Container +// Repository field, for the AWS ECR repo-creation call only - the DB column always keeps +// whatever the user typed, resolution happens fresh at every trigger (see HandlerService.go). +func (impl CiCdPipelineOrchestratorImpl) resolveDockerRepositoryForEcr(scope resourceQualifiers.Scope, dockerRepository string) (string, error) { + resolvedDockerRepository, _, err := impl.repoFieldVariableResolver.ResolveRepoFieldValue(scope, dockerRepository) + if err != nil { + return "", fmt.Errorf("error resolving docker repository: %w", err) + } + return resolvedDockerRepository, nil +} + +// resolveGitMaterialUrl resolves a possible @{{VAR_NAME}} reference in the git material URL. +// git_material.url in the DB always stays as the user typed it - only the value sent to +// git-sensor is resolved, so Application Templates keep the placeholder intact on clone. +func (impl CiCdPipelineOrchestratorImpl) resolveGitMaterialUrl(scope resourceQualifiers.Scope, url string) (string, error) { + resolvedUrl, _, err := impl.repoFieldVariableResolver.ResolveRepoFieldValue(scope, url) + if err != nil { + return "", fmt.Errorf("error resolving git material url: %w", err) + } + return resolvedUrl, nil +} + +// trackRepoFieldVariableMapping records (or clears) which scoped variable a Git Material's +// URL references, so a variable value change can find and re-sync every affected material. +func (impl CiCdPipelineOrchestratorImpl) trackRepoFieldVariableMapping(tx *pg.Tx, materialId int, url string, userId int32) error { + var variableNames []string + if variableName, ok := variables.ExtractVariableName(url); ok { + variableNames = []string{variableName} + } + entity := variableEntity.Entity{EntityType: variableEntity.EntityTypeGitMaterial, EntityId: materialId} + err := impl.variableEntityMappingService.UpdateVariablesForEntity(variableNames, entity, userId, tx) + if err != nil { + impl.logger.Errorw("error in tracking repo field variable mapping", "materialId", materialId, "err", err) + return err + } + return nil +} + +func (impl CiCdPipelineOrchestratorImpl) ResyncGitMaterialToSensor(materialId int) error { + material, err := impl.materialRepository.FindById(materialId) + if err != nil { + impl.logger.Errorw("error in finding material for resync", "materialId", materialId, "err", err) + return err + } + if !material.Active { + return nil + } + scope := resourceQualifiers.Scope{AppId: material.AppId} + return impl.updateRepositoryToGitSensor(material, "", false, scope) +} + func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *repository6.GitMaterial, - cloningMode string, createBackup bool) error { + cloningMode string, createBackup bool, scope resourceQualifiers.Scope) error { + // deletion only needs the Deleted flag - skip resolution so a since-deleted variable can't block a delete + resolvedUrl := material.Url + var err error + if material.Active { + resolvedUrl, err = impl.resolveGitMaterialUrl(scope, material.Url) + if err != nil { + return err + } + } sensorMaterial := &gitSensor.GitMaterial{ Name: material.Name, - Url: material.Url, + Url: resolvedUrl, Id: material.Id, GitProviderId: material.GitProviderId, CheckoutLocation: material.CheckoutPath, @@ -1523,12 +1626,16 @@ func (impl CiCdPipelineOrchestratorImpl) updateRepositoryToGitSensor(material *r return impl.GitSensorClient.UpdateRepo(ctx, sensorMaterial) } -func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial, cloningMode string) error { +func (impl CiCdPipelineOrchestratorImpl) addRepositoryToGitSensor(materials []*bean.GitMaterial, cloningMode string, scope resourceQualifiers.Scope) error { var sensorMaterials []*gitSensor.GitMaterial for _, material := range materials { + resolvedUrl, err := impl.resolveGitMaterialUrl(scope, material.Url) + if err != nil { + return err + } sensorMaterial := &gitSensor.GitMaterial{ Name: material.Name, - Url: material.Url, + Url: resolvedUrl, Id: material.Id, GitProviderId: material.GitProviderId, Deleted: false, @@ -1690,6 +1797,11 @@ func (impl CiCdPipelineOrchestratorImpl) updateMaterial(tx *pg.Tx, updateMateria return nil, err } + err = impl.trackRepoFieldVariableMapping(tx, currentMaterial.Id, currentMaterial.Url, updateMaterialDTO.UserId) + if err != nil { + return nil, err + } + err = impl.gitMaterialHistoryService.CreateMaterialHistory(tx, currentMaterial) return currentMaterial, nil @@ -1714,6 +1826,10 @@ func (impl CiCdPipelineOrchestratorImpl) createMaterial(tx *pg.Tx, inputMaterial impl.logger.Errorw("error in saving material", "material", material, "err", err) return nil, err } + err = impl.trackRepoFieldVariableMapping(tx, material.Id, material.Url, userId) + if err != nil { + return nil, err + } err = impl.gitMaterialHistoryService.CreateMaterialHistory(tx, material) return material, err } diff --git a/pkg/pipeline/GitMaterialVariableSyncService.go b/pkg/pipeline/GitMaterialVariableSyncService.go new file mode 100644 index 0000000000..ab307e5c94 --- /dev/null +++ b/pkg/pipeline/GitMaterialVariableSyncService.go @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pipeline + +import ( + "github.com/devtron-labs/devtron/pkg/variables" + variableEntity "github.com/devtron-labs/devtron/pkg/variables/repository" + "go.uber.org/zap" +) + +// GitMaterialVariableSyncService re-syncs every Git Material whose URL references a scoped +// variable to git-sensor. Called after the variable set changes (CreateVariables does a full +// delete-then-recreate of all variables, so any of them could be the one a material uses). +type GitMaterialVariableSyncService interface { + ResyncGitMaterialsForVariableChange() error +} + +type GitMaterialVariableSyncServiceImpl struct { + logger *zap.SugaredLogger + variableEntityMappingService variables.VariableEntityMappingService + ciCdPipelineOrchestrator CiCdPipelineOrchestrator +} + +func NewGitMaterialVariableSyncServiceImpl(logger *zap.SugaredLogger, + variableEntityMappingService variables.VariableEntityMappingService, + ciCdPipelineOrchestrator CiCdPipelineOrchestrator) *GitMaterialVariableSyncServiceImpl { + return &GitMaterialVariableSyncServiceImpl{ + logger: logger, + variableEntityMappingService: variableEntityMappingService, + ciCdPipelineOrchestrator: ciCdPipelineOrchestrator, + } +} + +func (impl *GitMaterialVariableSyncServiceImpl) ResyncGitMaterialsForVariableChange() error { + materialIds, err := impl.variableEntityMappingService.GetDistinctEntityIdsByType(variableEntity.EntityTypeGitMaterial) + if err != nil { + impl.logger.Errorw("error in finding git materials referencing scoped variables", "err", err) + return err + } + for _, materialId := range materialIds { + err = impl.ciCdPipelineOrchestrator.ResyncGitMaterialToSensor(materialId) + if err != nil { + // keep re-syncing the rest - one bad/stale mapping shouldn't block the others + impl.logger.Errorw("error re-syncing git material after variable change", "materialId", materialId, "err", err) + } + } + return nil +} diff --git a/pkg/resourceQualifiers/constants.go b/pkg/resourceQualifiers/constants.go index bcc16eb45a..6427f017cb 100644 --- a/pkg/resourceQualifiers/constants.go +++ b/pkg/resourceQualifiers/constants.go @@ -38,3 +38,14 @@ var SystemVariables = []SystemVariableName{ const AllExistingAndFutureNonProdEnvsInt = -1 const AllExistingAndFutureProdEnvsInt = -2 + +// IsSystemVariable reports whether name refers to a system variable, as opposed to a +// user-defined variable. +func IsSystemVariable(name string) bool { + for _, systemVariable := range SystemVariables { + if name == string(systemVariable) { + return true + } + } + return false +} diff --git a/pkg/resourceQualifiers/constants_test.go b/pkg/resourceQualifiers/constants_test.go new file mode 100644 index 0000000000..5d9b2ee017 --- /dev/null +++ b/pkg/resourceQualifiers/constants_test.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resourceQualifiers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestIsSystemVariable(t *testing.T) { + cases := []struct { + name string + variable string + want bool + }{ + {"fixed system variable", "DEVTRON_APP_NAME", true}, + {"another fixed system variable", "DEVTRON_ENV_NAME", true}, + {"user-defined variable", "FULL_GIT_URL", false}, + {"user-defined variable starting with DEVTRON", "DEVTRON_CUSTOM_THING", false}, + {"empty string", "", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + assert.Equal(t, c.want, IsSystemVariable(c.variable)) + }) + } +} diff --git a/pkg/variables/RepoFieldVariableResolver.go b/pkg/variables/RepoFieldVariableResolver.go new file mode 100644 index 0000000000..fc36a9bc74 --- /dev/null +++ b/pkg/variables/RepoFieldVariableResolver.go @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package variables + +import ( + "fmt" + "regexp" + "strings" + + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "go.uber.org/zap" +) + +// fullVariableReferenceRegex matches a field value that is exactly one @{{VAR_NAME}} token and nothing else. +var fullVariableReferenceRegex = regexp.MustCompile(`^@\{\{([A-Za-z0-9_]+)\}\}$`) + +// IsFullVariableReference reports whether fieldValue is exactly one @{{VAR_NAME}} token, so +// callers (e.g. request validation) can skip literal-value format checks without a DB lookup. +func IsFullVariableReference(fieldValue string) bool { + return fullVariableReferenceRegex.MatchString(fieldValue) +} + +// ExtractVariableName returns the variable name if fieldValue is a full @{{VAR_NAME}} +// reference, without doing any DB lookup - used to track which entities (e.g. Git Materials) +// reference which variables, independent of whether the reference currently resolves. +func ExtractVariableName(fieldValue string) (string, bool) { + matches := fullVariableReferenceRegex.FindStringSubmatch(fieldValue) + if matches == nil { + return "", false + } + return matches[1], true +} + +// RepoFieldVariableResolver resolves a single, full @{{VAR_NAME}} reference used in the Git +// Repository URL and Container Repository fields. Only user-defined variables are supported - +// scoped variables in this version only support Global scope, which fits these fields anyway +// since neither has any app/env/cluster context at its resolution point. +type RepoFieldVariableResolver interface { + // ResolveRepoFieldValue returns fieldValue unchanged if it isn't a variable reference, + // or its resolved value if it is a valid one. variableName is empty when nothing was resolved. + ResolveRepoFieldValue(scope resourceQualifiers.Scope, fieldValue string) (resolvedValue string, variableName string, err error) +} + +type RepoFieldVariableResolverImpl struct { + logger *zap.SugaredLogger + scopedVariableService ScopedVariableService +} + +func NewRepoFieldVariableResolverImpl(logger *zap.SugaredLogger, scopedVariableService ScopedVariableService) *RepoFieldVariableResolverImpl { + return &RepoFieldVariableResolverImpl{ + logger: logger, + scopedVariableService: scopedVariableService, + } +} + +func (impl *RepoFieldVariableResolverImpl) ResolveRepoFieldValue(scope resourceQualifiers.Scope, fieldValue string) (string, string, error) { + // plain literal value, nothing to resolve + if !strings.Contains(fieldValue, "@{{") { + return fieldValue, "", nil + } + + matches := fullVariableReferenceRegex.FindStringSubmatch(fieldValue) + if matches == nil { + // contains "@{{" but isn't a single, full @{{VAR_NAME}} token - not supported here + return "", "", fmt.Errorf("only a single full variable reference like @{{VAR_NAME}} is supported, not partial text or multiple variables") + } + variableName := matches[1] + + if resourceQualifiers.IsSystemVariable(variableName) { + return "", "", fmt.Errorf("system variable %q is not supported here, only user-defined variables are allowed", variableName) + } + + scopedVariables, err := impl.scopedVariableService.GetScopedVariables(scope, []string{variableName}, true) + if err != nil { + impl.logger.Errorw("error resolving repo field variable", "variableName", variableName, "scope", scope, "err", err) + return "", "", fmt.Errorf("error resolving variable %q: %w", variableName, err) + } + if len(scopedVariables) == 0 { + return "", "", fmt.Errorf("variable %q not found, or has no value defined", variableName) + } + + resolvedValue := scopedVariables[0].VariableValue.StringValue() + return resolvedValue, variableName, nil +} diff --git a/pkg/variables/RepoFieldVariableResolver_test.go b/pkg/variables/RepoFieldVariableResolver_test.go new file mode 100644 index 0000000000..be5f99d6ce --- /dev/null +++ b/pkg/variables/RepoFieldVariableResolver_test.go @@ -0,0 +1,134 @@ +/* + * Copyright (c) 2024. Devtron Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package variables + +import ( + "errors" + "testing" + + "github.com/devtron-labs/devtron/internal/util" + "github.com/devtron-labs/devtron/pkg/resourceQualifiers" + "github.com/devtron-labs/devtron/pkg/variables/models" + "github.com/stretchr/testify/assert" +) + +// fakeScopedVariableService resolves variables from a canned name->value map, and records +// the exact scope it was last called with so tests can assert env/cluster context is stripped. +type fakeScopedVariableService struct { + ScopedVariableService + definedVariables map[string]string + err error + lastScope resourceQualifiers.Scope +} + +func (f *fakeScopedVariableService) GetScopedVariables(scope resourceQualifiers.Scope, varNames []string, unmaskSensitiveData bool) ([]*models.ScopedVariableData, error) { + f.lastScope = scope + if f.err != nil { + return nil, f.err + } + scopedVariableData := make([]*models.ScopedVariableData, 0) + for _, name := range varNames { + if value, ok := f.definedVariables[name]; ok { + scopedVariableData = append(scopedVariableData, &models.ScopedVariableData{ + VariableName: name, + VariableValue: &models.VariableValue{Value: value}, + }) + } + } + return scopedVariableData, nil +} + +func newResolverForTest(t *testing.T, svc ScopedVariableService) *RepoFieldVariableResolverImpl { + logger, err := util.NewSugardLogger() + assert.Nil(t, err) + return NewRepoFieldVariableResolverImpl(logger, svc) +} + +func TestResolveRepoFieldValue(t *testing.T) { + svc := &fakeScopedVariableService{ + definedVariables: map[string]string{ + "FULL_GIT_URL": "https://github.com/org/repo.git", + }, + } + resolver := newResolverForTest(t, svc) + scope := resourceQualifiers.Scope{AppId: 1} + + t.Run("plain literal passes through unchanged", func(t *testing.T) { + resolved, varName, err := resolver.ResolveRepoFieldValue(scope, "https://github.com/org/literal.git") + assert.Nil(t, err) + assert.Equal(t, "https://github.com/org/literal.git", resolved) + assert.Empty(t, varName) + }) + + t.Run("full match resolves to defined value", func(t *testing.T) { + resolved, varName, err := resolver.ResolveRepoFieldValue(scope, "@{{FULL_GIT_URL}}") + assert.Nil(t, err) + assert.Equal(t, "https://github.com/org/repo.git", resolved) + assert.Equal(t, "FULL_GIT_URL", varName) + }) + + t.Run("undefined variable is rejected", func(t *testing.T) { + _, _, err := resolver.ResolveRepoFieldValue(scope, "@{{NOT_DEFINED}}") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "not found") + }) + + t.Run("partial match with surrounding text is rejected", func(t *testing.T) { + _, _, err := resolver.ResolveRepoFieldValue(scope, "https://github.com/@{{FULL_GIT_URL}}/repo.git") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "single full variable reference") + }) + + t.Run("multiple tokens in one field is rejected", func(t *testing.T) { + _, _, err := resolver.ResolveRepoFieldValue(scope, "@{{A}}@{{FULL_GIT_URL}}") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "single full variable reference") + }) + + t.Run("fixed system variable is rejected", func(t *testing.T) { + _, _, err := resolver.ResolveRepoFieldValue(scope, "@{{DEVTRON_APP_NAME}}") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "system variable") + }) + + t.Run("underlying resolution error is propagated", func(t *testing.T) { + failingSvc := &fakeScopedVariableService{err: errors.New("db unavailable")} + failingResolver := newResolverForTest(t, failingSvc) + _, _, err := failingResolver.ResolveRepoFieldValue(scope, "@{{FULL_GIT_URL}}") + assert.NotNil(t, err) + assert.Contains(t, err.Error(), "db unavailable") + }) +} + +func TestIsFullVariableReference(t *testing.T) { + assert.True(t, IsFullVariableReference("@{{FULL_GIT_URL}}")) + assert.False(t, IsFullVariableReference("https://github.com/@{{GIT_ORG}}/repo.git")) + assert.False(t, IsFullVariableReference("https://github.com/org/repo.git")) + assert.False(t, IsFullVariableReference("@{{A}}@{{B}}")) +} + +func TestExtractVariableName(t *testing.T) { + name, ok := ExtractVariableName("@{{FULL_GIT_URL}}") + assert.True(t, ok) + assert.Equal(t, "FULL_GIT_URL", name) + + _, ok = ExtractVariableName("https://github.com/org/repo.git") + assert.False(t, ok) + + _, ok = ExtractVariableName("https://github.com/@{{GIT_ORG}}/repo.git") + assert.False(t, ok) +} diff --git a/pkg/variables/VariableEntityMappingService.go b/pkg/variables/VariableEntityMappingService.go index e6b1517f69..64a6e8ba07 100644 --- a/pkg/variables/VariableEntityMappingService.go +++ b/pkg/variables/VariableEntityMappingService.go @@ -32,6 +32,10 @@ type VariableEntityMappingService interface { GetAllMappingsForEntities(entities []repository.Entity) (map[repository.Entity][]string, error) GetLiveVariableUsage(variableNames []string) ([]*models.VariableUsage, error) DeleteMappingsForEntities(entities []repository.Entity, userId int32, tx *pg.Tx) error + // GetDistinctEntityIdsByType returns every entity id of entityType that references any + // variable - e.g. every Git Material with a @{{VAR_NAME}} URL, for re-sync after a + // variable value changes. + GetDistinctEntityIdsByType(entityType repository.EntityType) ([]int, error) } type VariableEntityMappingServiceImpl struct { @@ -137,6 +141,15 @@ func (impl VariableEntityMappingServiceImpl) GetAllMappingsForEntities(entities return entityIdToVariableNames, nil } +func (impl VariableEntityMappingServiceImpl) GetDistinctEntityIdsByType(entityType repository.EntityType) ([]int, error) { + entityIds, err := impl.variableEntityMappingRepository.GetDistinctEntityIdsByType(entityType) + if err != nil { + impl.logger.Errorw("error in getting distinct entity ids by type", "entityType", entityType, "err", err) + return nil, err + } + return entityIds, nil +} + func getUsageTypeForEntityType(entityType repository.EntityType) models.VariableUsageType { switch entityType { case repository.EntityTypeDeploymentTemplateAppLevel, repository.EntityTypeDeploymentTemplateEnvLevel: diff --git a/pkg/variables/repository/VariableEntityDTO.go b/pkg/variables/repository/VariableEntityDTO.go index d9eacf705e..400eb8bc48 100644 --- a/pkg/variables/repository/VariableEntityDTO.go +++ b/pkg/variables/repository/VariableEntityDTO.go @@ -45,6 +45,9 @@ const ( EntityTypeConfigMapEnvLevel EntityType = 5 EntityTypeSecretAppLevel EntityType = 6 EntityTypeSecretEnvLevel EntityType = 7 + // EntityTypeGitMaterial tracks which Git Materials reference a scoped variable in their + // URL, so a variable value change can re-sync the affected materials to git-sensor. + EntityTypeGitMaterial EntityType = 9 ) func GetEntity(entityId int, entityType EntityType) Entity { diff --git a/pkg/variables/repository/VariableEntityMappingRepository.go b/pkg/variables/repository/VariableEntityMappingRepository.go index de2ce409c1..97c5bbff0e 100644 --- a/pkg/variables/repository/VariableEntityMappingRepository.go +++ b/pkg/variables/repository/VariableEntityMappingRepository.go @@ -31,6 +31,10 @@ type VariableEntityMappingRepository interface { SaveVariableEntityMappings(tx *pg.Tx, mappings []*VariableEntityMapping) error DeleteAllVariablesForEntities(tx *pg.Tx, entities []Entity, userId int32) error DeleteVariablesForEntity(tx *pg.Tx, variableIDs []string, entity Entity, userId int32) error + // GetDistinctEntityIdsByType returns every non-deleted entity id of entityType that + // references any variable - used to find every Git Material that needs re-syncing + // after a scoped variable's value changes. + GetDistinctEntityIdsByType(entityType EntityType) ([]int, error) } // VariableUsageRow is one live reference of a variable, resolved to the app/env/pipeline it belongs to. @@ -172,6 +176,20 @@ func (impl *VariableEntityMappingRepositoryImpl) DeleteVariablesForEntity(tx *pg return nil } +func (impl *VariableEntityMappingRepositoryImpl) GetDistinctEntityIdsByType(entityType EntityType) ([]int, error) { + var entityIds []int + err := impl.dbConnection.Model((*VariableEntityMapping)(nil)). + ColumnExpr("DISTINCT entity_id"). + Where("entity_type = ?", entityType). + Where("is_deleted = ?", false). + Select(&entityIds) + if err != nil && err != pg.ErrNoRows { + impl.logger.Errorw("err in getting distinct entity ids by type", "entityType", entityType, "err", err) + return nil, err + } + return entityIds, nil +} + func (impl *VariableEntityMappingRepositoryImpl) DeleteAllVariablesForEntities(tx *pg.Tx, entities []Entity, userId int32) error { var connection orm.DB diff --git a/wire_gen.go b/wire_gen.go index a3bc5f5b69..7f2744bbfd 100644 --- a/wire_gen.go +++ b/wire_gen.go @@ -672,6 +672,7 @@ func InitializeApp() (*App, error) { materialRepositoryImpl := repository23.NewMaterialRepositoryImpl(db) gitMaterialReadServiceImpl := read15.NewGitMaterialReadServiceImpl(sugaredLogger, materialRepositoryImpl) appCrudOperationServiceImpl := app2.NewAppCrudOperationServiceImpl(appLabelRepositoryImpl, sugaredLogger, appRepositoryImpl, userRepositoryImpl, installedAppRepositoryImpl, genericNoteServiceImpl, installedAppDBServiceImpl, crudOperationServiceConfig, dbMigrationServiceImpl, gitMaterialReadServiceImpl) + repoFieldVariableResolverImpl := variables.NewRepoFieldVariableResolverImpl(sugaredLogger, scopedVariableServiceImpl) imageTagRepositoryImpl := repository2.NewImageTagRepository(db, sugaredLogger) customTagServiceImpl := pipeline.NewCustomTagService(sugaredLogger, imageTagRepositoryImpl) clientConfig, err := gitSensor.GetConfig() @@ -699,7 +700,7 @@ func InitializeApp() (*App, error) { deploymentTemplateHistoryServiceImpl := deploymentTemplate.NewDeploymentTemplateHistoryServiceImpl(sugaredLogger, deploymentTemplateHistoryRepositoryImpl, pipelineRepositoryImpl, chartRepositoryImpl, userServiceImpl, cdWorkflowRepositoryImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl) chartReadServiceImpl := read16.NewChartReadServiceImpl(sugaredLogger, chartRepositoryImpl, deploymentConfigServiceImpl, deployedAppMetricsServiceImpl, gitOpsConfigReadServiceImpl, chartRefReadServiceImpl) chartServiceImpl := chart.NewChartServiceImpl(chartRepositoryImpl, sugaredLogger, chartTemplateServiceImpl, chartRepoRepositoryImpl, appRepositoryImpl, mergeUtil, envConfigOverrideRepositoryImpl, pipelineConfigRepositoryImpl, environmentRepositoryImpl, deploymentTemplateHistoryServiceImpl, scopedVariableManagerImpl, deployedAppMetricsServiceImpl, chartRefServiceImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, envConfigOverrideReadServiceImpl, chartReadServiceImpl) - ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl, environmentVariables) + ciCdPipelineOrchestratorImpl := pipeline.NewCiCdPipelineOrchestrator(appRepositoryImpl, sugaredLogger, materialRepositoryImpl, pipelineRepositoryImpl, ciPipelineRepositoryImpl, ciPipelineMaterialRepositoryImpl, cdWorkflowRepositoryImpl, clientImpl, ciCdConfig, appWorkflowRepositoryImpl, environmentRepositoryImpl, attributesServiceImpl, appCrudOperationServiceImpl, userAuthServiceImpl, prePostCdScriptHistoryServiceImpl, pipelineStageServiceImpl, gitMaterialHistoryServiceImpl, ciPipelineHistoryServiceImpl, ciTemplateReadServiceImpl, ciTemplateServiceImpl, dockerArtifactStoreRepositoryImpl, ciArtifactRepositoryImpl, configMapServiceImpl, customTagServiceImpl, genericNoteServiceImpl, chartServiceImpl, transactionUtilImpl, gitOpsConfigReadServiceImpl, deploymentConfigServiceImpl, deploymentConfigReadServiceImpl, chartReadServiceImpl, environmentVariables, repoFieldVariableResolverImpl, variableEntityMappingServiceImpl) pluginInputVariableParserImpl := pipeline.NewPluginInputVariableParserImpl(sugaredLogger, dockerRegistryConfigImpl, customTagServiceImpl) ciServiceImpl := pipeline.NewCiServiceImpl(sugaredLogger, workFlowStageStatusServiceImpl, eventRESTClientImpl, eventSimpleFactoryImpl, ciWorkflowRepositoryImpl, ciPipelineRepositoryImpl, transactionUtilImpl, workflowStatusLatestServiceImpl) ciLogServiceImpl, err := pipeline.NewCiLogServiceImpl(sugaredLogger, k8sServiceImpl) @@ -707,7 +708,7 @@ func InitializeApp() (*App, error) { return nil, err } blobStorageConfigServiceImpl := pipeline.NewBlobStorageConfigServiceImpl(sugaredLogger, k8sServiceImpl, ciCdConfig) - handlerServiceImpl := trigger.NewHandlerServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl, runnable, workflowTriggerAuditServiceImpl) + handlerServiceImpl := trigger.NewHandlerServiceImpl(sugaredLogger, workflowServiceImpl, ciPipelineMaterialRepositoryImpl, ciPipelineRepositoryImpl, ciArtifactRepositoryImpl, pipelineStageServiceImpl, userServiceImpl, ciTemplateReadServiceImpl, appCrudOperationServiceImpl, environmentRepositoryImpl, appRepositoryImpl, scopedVariableManagerImpl, repoFieldVariableResolverImpl, customTagServiceImpl, ciCdPipelineOrchestratorImpl, attributesServiceImpl, pluginInputVariableParserImpl, globalPluginServiceImpl, ciServiceImpl, ciWorkflowRepositoryImpl, clientImpl, ciLogServiceImpl, blobStorageConfigServiceImpl, clusterServiceImplExtended, environmentServiceImpl, k8sServiceImpl, runnable, workflowTriggerAuditServiceImpl) gitWebhookServiceImpl := gitWebhook.NewGitWebhookServiceImpl(sugaredLogger, gitWebhookRepositoryImpl, handlerServiceImpl) gitWebhookRestHandlerImpl := restHandler.NewGitWebhookRestHandlerImpl(sugaredLogger, gitWebhookServiceImpl) ecrConfig, err := pipeline.GetEcrConfig() @@ -1076,7 +1077,8 @@ func InitializeApp() (*App, error) { rbacRoleServiceImpl := user.NewRbacRoleServiceImpl(sugaredLogger, rbacRoleDataRepositoryImpl) rbacRoleRestHandlerImpl := user2.NewRbacRoleHandlerImpl(sugaredLogger, validate, rbacRoleServiceImpl, userServiceImpl, enforcerImpl, enforcerUtilImpl) rbacRoleRouterImpl := user2.NewRbacRoleRouterImpl(sugaredLogger, validate, rbacRoleRestHandlerImpl) - scopedVariableRestHandlerImpl := scopedVariable.NewScopedVariableRestHandlerImpl(sugaredLogger, userServiceImpl, validate, pipelineBuilderImpl, enforcerUtilImpl, enforcerImpl, scopedVariableServiceImpl) + gitMaterialVariableSyncServiceImpl := pipeline.NewGitMaterialVariableSyncServiceImpl(sugaredLogger, variableEntityMappingServiceImpl, ciCdPipelineOrchestratorImpl) + scopedVariableRestHandlerImpl := scopedVariable.NewScopedVariableRestHandlerImpl(sugaredLogger, userServiceImpl, validate, pipelineBuilderImpl, enforcerUtilImpl, enforcerImpl, scopedVariableServiceImpl, gitMaterialVariableSyncServiceImpl, runnable) scopedVariableRouterImpl := router.NewScopedVariableRouterImpl(scopedVariableRestHandlerImpl) ciTriggerCronConfig, err := cron2.GetCiTriggerCronConfig() if err != nil { From 5e957a5e34301372de1b0db6e886a303371cabd5 Mon Sep 17 00:00:00 2001 From: Shivam-nagar23 Date: Mon, 21 Sep 2026 17:07:37 +0530 Subject: [PATCH 11/12] chore: update devtron-services deps to main for v0.47.0 Points common-lib and authenticator at devtron-services main (f0b3fba15081b8d40a795928aaad5df55473c681, the v0.47.0 RC merge). Co-Authored-By: Claude Opus 5 (1M context) --- go.mod | 4 ++-- go.sum | 8 ++++---- vendor/modules.txt | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index e2cc19dca3..500ac628bb 100644 --- a/go.mod +++ b/go.mod @@ -341,7 +341,7 @@ replace ( // Keep the left side unversioned: a versioned LHS stops matching whenever the // require line is bumped, which silently un-pins this and breaks the linux build. github.com/cyphar/filepath-securejoin => github.com/cyphar/filepath-securejoin v0.3.6 // indirect - github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260803100101-66fcb35e4b0e - github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260803100101-66fcb35e4b0e + github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260921111017-f0b3fba15081 + github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260921111017-f0b3fba15081 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 => go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 ) diff --git a/go.sum b/go.sum index f22eaecf9a..12d371f020 100644 --- a/go.sum +++ b/go.sum @@ -237,10 +237,10 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc h1:VRRKCwnzq github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= github.com/devtron-labs/argo-workflows/v3 v3.5.13 h1:3pINq0gXOSeTw2z/vYe+j80lRpSN5Rp/8mfQORh8SmU= github.com/devtron-labs/argo-workflows/v3 v3.5.13/go.mod h1:/vqxcovDPT4zqr4DjR5v7CF8ggpY1l3TSa2CIG3jmjA= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260803100101-66fcb35e4b0e h1:mfAbqKv+CswkCah0JCSzeTJThvjaNBt51TunM8lBsPU= -github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260803100101-66fcb35e4b0e/go.mod h1:m/dMed6iiZQ/4iOJ6kIVWXQ+T71rJjU7DnCr9RcYh1w= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260803100101-66fcb35e4b0e h1:gh/iIVC9FNpPE8C72duWZGNgwgE2jrYonpYDvty2cFM= -github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260803100101-66fcb35e4b0e/go.mod h1:QZfnuGy0xqrYWhq3jCRe6fDDwIsrpbjAR2PXNrh0su8= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260921111017-f0b3fba15081 h1:BPkaewVaxMTnIAHcxwsWgebl5GV7uc6R/a0tCBoTZn0= +github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260921111017-f0b3fba15081/go.mod h1:m/dMed6iiZQ/4iOJ6kIVWXQ+T71rJjU7DnCr9RcYh1w= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260921111017-f0b3fba15081 h1:niegDwFaPhH5b+7UP2QuGdR6rB43hFYqNY1pJ6ZbXQw= +github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260921111017-f0b3fba15081/go.mod h1:QZfnuGy0xqrYWhq3jCRe6fDDwIsrpbjAR2PXNrh0su8= github.com/devtron-labs/go-bitbucket v0.9.60-beta h1:VEx1jvDgdtDPS6A1uUFoaEi0l1/oLhbr+90xOwr6sDU= github.com/devtron-labs/go-bitbucket v0.9.60-beta/go.mod h1:GnuiCesvh8xyHeMCb+twm8lBR/kQzJYSKL28ZfObp1Y= github.com/devtron-labs/protos v0.0.3-0.20250323220609-ecf8a0f7305e h1:U6UdYbW8a7xn5IzFPd8cywjVVPfutGJCudjePAfL/Hs= diff --git a/vendor/modules.txt b/vendor/modules.txt index 4ee86c28d3..eae973ecd9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -523,7 +523,7 @@ github.com/davecgh/go-spew/spew # github.com/deckarep/golang-set v1.8.0 ## explicit; go 1.17 github.com/deckarep/golang-set -# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260803100101-66fcb35e4b0e +# github.com/devtron-labs/authenticator v0.4.35-0.20240809073103-6e11da8083f8 => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260921111017-f0b3fba15081 ## explicit; go 1.25.0 github.com/devtron-labs/authenticator/apiToken github.com/devtron-labs/authenticator/client @@ -531,7 +531,7 @@ github.com/devtron-labs/authenticator/jwt github.com/devtron-labs/authenticator/middleware github.com/devtron-labs/authenticator/oidc github.com/devtron-labs/authenticator/password -# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260803100101-66fcb35e4b0e +# github.com/devtron-labs/common-lib v0.18.1-0.20241001061923-eda545dc839e => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260921111017-f0b3fba15081 ## explicit; go 1.25.0 github.com/devtron-labs/common-lib/async github.com/devtron-labs/common-lib/blob-storage @@ -2697,5 +2697,5 @@ xorm.io/xorm/names xorm.io/xorm/schemas xorm.io/xorm/tags # github.com/cyphar/filepath-securejoin => github.com/cyphar/filepath-securejoin v0.3.6 -# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260803100101-66fcb35e4b0e -# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260803100101-66fcb35e4b0e +# github.com/devtron-labs/authenticator => github.com/devtron-labs/devtron-services/authenticator v0.0.0-20260921111017-f0b3fba15081 +# github.com/devtron-labs/common-lib => github.com/devtron-labs/devtron-services/common-lib v0.0.0-20260921111017-f0b3fba15081 From c83900e0c7b76647f3c450c1a2b30dae33465028 Mon Sep 17 00:00:00 2001 From: Shivam-nagar23 Date: Mon, 21 Sep 2026 17:30:01 +0530 Subject: [PATCH 12/12] chore(sql): resolve migration 36504600 collision and make argo/flux rbac idempotent Two different migrations shared version 36504600 on the release branch: add_ci_workflow_active_status_index and dev_admin_app_lifecycle. The latter is already released on main at that number, so the CI index migration (RC-only) is renumbered to 36904600, the next free slot. Also adds ON CONFLICT ("entity","access_type","role") DO NOTHING to both INSERTs in 36804600_argo_flux_rbac.up.sql. Both rbac_policy_data and rbac_role_data declare UNIQUE (entity, access_type, role), so a re-run previously failed on the unique constraint instead of being a no-op. Co-Authored-By: Claude Opus 5 (1M context) --- scripts/sql/36804600_argo_flux_rbac.up.sql | 6 ++++-- ...> 36904600_add_ci_workflow_active_status_index.down.sql} | 0 ... => 36904600_add_ci_workflow_active_status_index.up.sql} | 0 3 files changed, 4 insertions(+), 2 deletions(-) rename scripts/sql/{36504600_add_ci_workflow_active_status_index.down.sql => 36904600_add_ci_workflow_active_status_index.down.sql} (100%) rename scripts/sql/{36504600_add_ci_workflow_active_status_index.up.sql => 36904600_add_ci_workflow_active_status_index.up.sql} (100%) diff --git a/scripts/sql/36804600_argo_flux_rbac.up.sql b/scripts/sql/36804600_argo_flux_rbac.up.sql index c7ce5e8a0c..0f10c65062 100644 --- a/scripts/sql/36804600_argo_flux_rbac.up.sql +++ b/scripts/sql/36804600_argo_flux_rbac.up.sql @@ -60,7 +60,8 @@ VALUES "act": { "value": "get", "indexKeyMap": {} }, "obj": { "value": "%", "indexKeyMap": { "0": "EnvObj" } } } ] -}','now()','1','now()','1',true,false); +}','now()','1','now()','1',true,false) +ON CONFLICT ("entity", "access_type", "role") DO NOTHING; INSERT INTO "public"."rbac_role_data" ("entity","access_type","role","role_display_name","role_description","role_data", @@ -108,4 +109,5 @@ VALUES "action": { "value": "admin", "indexKeyMap": {} }, "entity": { "value": "%", "indexKeyMap": { "0": "Entity" } }, "accessType": { "value": "flux-app", "indexKeyMap": {} } -}','now()','1','now()','1',true,false); \ No newline at end of file +}','now()','1','now()','1',true,false) +ON CONFLICT ("entity", "access_type", "role") DO NOTHING; diff --git a/scripts/sql/36504600_add_ci_workflow_active_status_index.down.sql b/scripts/sql/36904600_add_ci_workflow_active_status_index.down.sql similarity index 100% rename from scripts/sql/36504600_add_ci_workflow_active_status_index.down.sql rename to scripts/sql/36904600_add_ci_workflow_active_status_index.down.sql diff --git a/scripts/sql/36504600_add_ci_workflow_active_status_index.up.sql b/scripts/sql/36904600_add_ci_workflow_active_status_index.up.sql similarity index 100% rename from scripts/sql/36504600_add_ci_workflow_active_status_index.up.sql rename to scripts/sql/36904600_add_ci_workflow_active_status_index.up.sql