Issue #1219 upgrade deploy pipeline (#1220)

Manangka · web-flow · commit 7b2262487b75 · 2024-09-16T15:13:38.000Z
Fixes #1219 # Description In this PR the Deploy pipeline and pixi pipeline have been updated to use the docker build images. Further more a docker image and docker version parameter have been created on root level, making it easy to update those # Checklist  - [x] Links to correct issue - [ ] Update changelog, if changes affect users - [x] PR title starts with ``Issue #nr``, e.g. ``Issue #737`` - [ ] Unit tests were added - [ ] **If feature added**: Added/extended example
diff --git a/.teamcity/Deploy/DeployProject.kt b/.teamcity/Deploy/DeployProject.kt
@@ -1,6 +1,8 @@
 package Deploy
 
 import jetbrains.buildServer.configs.kotlin.*
+import jetbrains.buildServer.configs.kotlin.buildFeatures.dockerSupport
+import jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep
 import jetbrains.buildServer.configs.kotlin.buildSteps.powerShell
 import jetbrains.buildServer.configs.kotlin.buildSteps.script
 import jetbrains.buildServer.configs.kotlin.triggers.vcs
@@ -42,11 +44,19 @@ object BuildPackage : BuildType({
                 pixi run --environment default --frozen python -m build
             """.trimIndent()
             formatStderrAsError = true
+            dockerImage = "%DockerContainer%:%DockerVersion%"
+            dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
+            dockerRunParameters = """--cpus=4 --memory=16g"""
+            dockerPull = true
         }
     }
 
-    requirements {
-        equals("env.OS", "Windows_NT")
+    features {
+        dockerSupport {
+            loginToRegistry = on {
+                dockerRegistryId = "PROJECT_EXT_134"
+            }
+        }
     }
 })
 
@@ -72,11 +82,15 @@ object BuildPages : BuildType({
             name = "Build documentation"
             workingDir = "imod-python"
             scriptContent = """
-                set Path=%system.teamcity.build.checkoutDir%\modflow6;%env.Path% 
+                SET PATH=%%PATH%%;%system.teamcity.build.checkoutDir%\modflow6
                 
                 pixi run --environment default --frozen docs
             """.trimIndent()
             formatStderrAsError = true
+            dockerImage = "%DockerContainer%:%DockerVersion%"
+            dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
+            dockerRunParameters = """--cpus=8 --memory=32g"""
+            dockerPull = true
         }
     }
 
@@ -105,8 +119,12 @@ object BuildPages : BuildType({
         }
     }
 
-    requirements {
-        equals("env.OS", "Windows_NT")
+    features {
+        dockerSupport {
+            loginToRegistry = on {
+                dockerRegistryId = "PROJECT_EXT_134"
+            }
+        }
     }
 })
 
@@ -147,15 +165,23 @@ object CreateGitHubRelease : BuildType({
                     pixi run --environment default --frozen gh release create ${'$'}tag --verify-tag --notes "See https://deltares.github.io/imod-python/api/changelog.html"
                 """.trimIndent()
             }
+            param("plugin.docker.imagePlatform", "windows")
+            param("plugin.docker.pull.enabled", "true")
+            param("plugin.docker.imageId", "%DockerContainer%:%DockerVersion%")
+            param("plugin.docker.run.parameters", "--cpus=4 --memory=16g")
         }
     }
 
     failureConditions {
         errorMessage = true
     }
 
-    requirements {
-        equals("env.OS", "Windows_NT")
+    features {
+        dockerSupport {
+            loginToRegistry = on {
+                dockerRegistryId = "PROJECT_EXT_134"
+            }
+        }
     }
 })
 
@@ -188,6 +214,10 @@ object DeployPackage : BuildType({
                 pixi run --frozen twine check ../dist/*
                 pixi run --frozen twine upload ../dist/*
             """.trimIndent()
+            dockerImage = "%DockerContainer%:%DockerVersion%"
+            dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
+            dockerRunParameters = """--cpus=4 --memory=16g"""
+            dockerPull = true
         }
     }
 
@@ -203,8 +233,12 @@ object DeployPackage : BuildType({
         }
     }
 
-    requirements {
-        equals("env.OS", "Windows_NT")
+    features {
+        dockerSupport {
+            loginToRegistry = on {
+                dockerRegistryId = "PROJECT_EXT_134"
+            }
+        }
     }
 })
 
@@ -260,6 +294,10 @@ object DeployPages : BuildType({
                 git push origin gh-pages
             """.trimIndent()
             formatStderrAsError = true
+            dockerImage = "%DockerContainer%:%DockerVersion%"
+            dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
+            dockerRunParameters = """--cpus=4 --memory=16g"""
+            dockerPull = true
         }
     }
 
@@ -276,8 +314,12 @@ object DeployPages : BuildType({
         }
     }
 
-    requirements {
-        equals("env.OS", "Windows_NT")
+    features {
+        dockerSupport {
+            loginToRegistry = on {
+                dockerRegistryId = "PROJECT_EXT_134"
+            }
+        }
     }
 })
 
diff --git a/.teamcity/Pixi/PixiProject.kt b/.teamcity/Pixi/PixiProject.kt
@@ -3,6 +3,7 @@ package Pixi
 import jetbrains.buildServer.configs.kotlin.AbsoluteId
 import jetbrains.buildServer.configs.kotlin.BuildType
 import jetbrains.buildServer.configs.kotlin.Project
+import jetbrains.buildServer.configs.kotlin.buildFeatures.dockerSupport
 import jetbrains.buildServer.configs.kotlin.buildSteps.powerShell
 import jetbrains.buildServer.configs.kotlin.triggers.schedule
 
@@ -64,6 +65,10 @@ object UpdateDependencies : BuildType({
                 """.trimIndent()
             }
             noProfile = false
+            param("plugin.docker.imagePlatform", "windows")
+            param("plugin.docker.pull.enabled", "true")
+            param("plugin.docker.imageId", "%DockerContainer%:%DockerVersion%")
+            param("plugin.docker.run.parameters", "--cpus=4 --memory=16g")
         }
     }
 
@@ -81,4 +86,12 @@ object UpdateDependencies : BuildType({
     failureConditions {
         errorMessage = true
     }
+
+    features {
+        dockerSupport {
+            loginToRegistry = on {
+                dockerRegistryId = "PROJECT_EXT_134"
+            }
+        }
+    }
 })
diff --git a/.teamcity/Templates/ExamplesTemplate.kt b/.teamcity/Templates/ExamplesTemplate.kt
@@ -31,7 +31,7 @@ object ExamplesTemplate : Template({
                 pixi run --environment default --frozen examples
             """.trimIndent()
             formatStderrAsError = true
-            dockerImage = "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1"
+            dockerImage = "%DockerContainer%:%DockerVersion%"
             dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
             dockerRunParameters = """--cpus=8 --memory=32g"""
             dockerPull = true
diff --git a/.teamcity/Templates/LintTemplate.kt b/.teamcity/Templates/LintTemplate.kt
@@ -26,7 +26,7 @@ object LintTemplate : Template({
                     pixi run --environment default --frozen lint 
                 """.trimIndent()
             formatStderrAsError = true
-            dockerImage = "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1"
+            dockerImage = "%DockerContainer%:%DockerVersion%"
             dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
             dockerRunParameters = """--cpus=4 --memory=16g"""
             dockerPull = true
diff --git a/.teamcity/Templates/MyPyTemplate.kt b/.teamcity/Templates/MyPyTemplate.kt
@@ -31,7 +31,7 @@ object MyPyTemplate : Template({
                     pixi run --environment default --frozen mypy
                 """.trimIndent()
             formatStderrAsError = true
-            dockerImage = "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1"
+            dockerImage = "%DockerContainer%:%DockerVersion%"
             dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
             dockerRunParameters = """--cpus=4 --memory=16g"""
             dockerPull = true
diff --git a/.teamcity/Templates/PipPythonTemplate.kt b/.teamcity/Templates/PipPythonTemplate.kt
@@ -27,7 +27,7 @@ object PipPythonTemplate : Template({
                     pixi run --environment %python_env% --frozen test_import
                 """.trimIndent()
             formatStderrAsError = true
-            dockerImage = "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1"
+            dockerImage = "%DockerContainer%:%DockerVersion%"
             dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
             dockerRunParameters = """--cpus=4 --memory=16g"""
             dockerPull = true
diff --git a/.teamcity/Templates/UnitTestsTemplate.kt b/.teamcity/Templates/UnitTestsTemplate.kt
@@ -34,7 +34,7 @@ object UnitTestsTemplate : Template({
                 pixi run --environment default --frozen unittests
             """.trimIndent()
             formatStderrAsError = true
-            dockerImage = "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1"
+            dockerImage = "%DockerContainer%:%DockerVersion%"
             dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows
             dockerRunParameters = """--cpus=8 --memory=32g"""
             dockerPull = true
@@ -58,7 +58,7 @@ object UnitTestsTemplate : Template({
                 """.trimIndent()
             }
             param("plugin.docker.imagePlatform", "windows")
-            param("plugin.docker.imageId", "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1")
+            param("plugin.docker.imageId", "%DockerContainer%:%DockerVersion%")
             param("plugin.docker.pull.enabled", "true")
         }
     }
diff --git a/.teamcity/_Self/MainProject.kt b/.teamcity/_Self/MainProject.kt
@@ -15,6 +15,11 @@ import jetbrains.buildServer.configs.kotlin.projectFeatures.projectReportTab
 import jetbrains.buildServer.configs.kotlin.triggers.vcs
 
 object MainProject : Project({
+    params {
+        param("DockerContainer", "containers.deltares.nl/hydrology_product_line_imod/windows-pixi")
+        param("DockerVersion", "v0.26.1")
+    }
+
     buildType(Lint)
     buildType(MyPy)
     buildType(UnitTests)

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,8 @@`
`1`	`1`	`package Deploy`
`2`	`2`
`3`	`3`	`import jetbrains.buildServer.configs.kotlin.*`
	`4`	`+import jetbrains.buildServer.configs.kotlin.buildFeatures.dockerSupport`
	`5`	`+import jetbrains.buildServer.configs.kotlin.buildSteps.ScriptBuildStep`
`4`	`6`	`import jetbrains.buildServer.configs.kotlin.buildSteps.powerShell`
`5`	`7`	`import jetbrains.buildServer.configs.kotlin.buildSteps.script`
`6`	`8`	`import jetbrains.buildServer.configs.kotlin.triggers.vcs`
`@@ -42,11 +44,19 @@ object BuildPackage : BuildType({`
`42`	`44`	`pixi run --environment default --frozen python -m build`
`43`	`45`	`""".trimIndent()`
`44`	`46`	`formatStderrAsError = true`
	`47`	`+ dockerImage = "%DockerContainer%:%DockerVersion%"`
	`48`	`+ dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows`
	`49`	`+ dockerRunParameters = """--cpus=4 --memory=16g"""`
	`50`	`+ dockerPull = true`
`45`	`51`	`}`
`46`	`52`	`}`
`47`	`53`
`48`		`- requirements {`
`49`		`- equals("env.OS", "Windows_NT")`
	`54`	`+ features {`
	`55`	`+ dockerSupport {`
	`56`	`+ loginToRegistry = on {`
	`57`	`+ dockerRegistryId = "PROJECT_EXT_134"`
	`58`	`+ }`
	`59`	`+ }`
`50`	`60`	`}`
`51`	`61`	`})`
`52`	`62`
`@@ -72,11 +82,15 @@ object BuildPages : BuildType({`
`72`	`82`	`name = "Build documentation"`
`73`	`83`	`workingDir = "imod-python"`
`74`	`84`	`scriptContent = """`
`75`		`- set Path=%system.teamcity.build.checkoutDir%\modflow6;%env.Path%`
	`85`	`+ SET PATH=%%PATH%%;%system.teamcity.build.checkoutDir%\modflow6`
`76`	`86`
`77`	`87`	`pixi run --environment default --frozen docs`
`78`	`88`	`""".trimIndent()`
`79`	`89`	`formatStderrAsError = true`
	`90`	`+ dockerImage = "%DockerContainer%:%DockerVersion%"`
	`91`	`+ dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows`
	`92`	`+ dockerRunParameters = """--cpus=8 --memory=32g"""`
	`93`	`+ dockerPull = true`
`80`	`94`	`}`
`81`	`95`	`}`
`82`	`96`
`@@ -105,8 +119,12 @@ object BuildPages : BuildType({`
`105`	`119`	`}`
`106`	`120`	`}`
`107`	`121`
`108`		`- requirements {`
`109`		`- equals("env.OS", "Windows_NT")`
	`122`	`+ features {`
	`123`	`+ dockerSupport {`
	`124`	`+ loginToRegistry = on {`
	`125`	`+ dockerRegistryId = "PROJECT_EXT_134"`
	`126`	`+ }`
	`127`	`+ }`
`110`	`128`	`}`
`111`	`129`	`})`
`112`	`130`
`@@ -147,15 +165,23 @@ object CreateGitHubRelease : BuildType({`
`147`	`165`	`pixi run --environment default --frozen gh release create ${'$'}tag --verify-tag --notes "See https://deltares.github.io/imod-python/api/changelog.html"`
`148`	`166`	`""".trimIndent()`
`149`	`167`	`}`
	`168`	`+ param("plugin.docker.imagePlatform", "windows")`
	`169`	`+ param("plugin.docker.pull.enabled", "true")`
	`170`	`+ param("plugin.docker.imageId", "%DockerContainer%:%DockerVersion%")`
	`171`	`+ param("plugin.docker.run.parameters", "--cpus=4 --memory=16g")`
`150`	`172`	`}`
`151`	`173`	`}`
`152`	`174`
`153`	`175`	`failureConditions {`
`154`	`176`	`errorMessage = true`
`155`	`177`	`}`
`156`	`178`
`157`		`- requirements {`
`158`		`- equals("env.OS", "Windows_NT")`
	`179`	`+ features {`
	`180`	`+ dockerSupport {`
	`181`	`+ loginToRegistry = on {`
	`182`	`+ dockerRegistryId = "PROJECT_EXT_134"`
	`183`	`+ }`
	`184`	`+ }`
`159`	`185`	`}`
`160`	`186`	`})`
`161`	`187`
`@@ -188,6 +214,10 @@ object DeployPackage : BuildType({`
`188`	`214`	`pixi run --frozen twine check ../dist/*`
`189`	`215`	`pixi run --frozen twine upload ../dist/*`
`190`	`216`	`""".trimIndent()`
	`217`	`+ dockerImage = "%DockerContainer%:%DockerVersion%"`
	`218`	`+ dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows`
	`219`	`+ dockerRunParameters = """--cpus=4 --memory=16g"""`
	`220`	`+ dockerPull = true`
`191`	`221`	`}`
`192`	`222`	`}`
`193`	`223`
`@@ -203,8 +233,12 @@ object DeployPackage : BuildType({`
`203`	`233`	`}`
`204`	`234`	`}`
`205`	`235`
`206`		`- requirements {`
`207`		`- equals("env.OS", "Windows_NT")`
	`236`	`+ features {`
	`237`	`+ dockerSupport {`
	`238`	`+ loginToRegistry = on {`
	`239`	`+ dockerRegistryId = "PROJECT_EXT_134"`
	`240`	`+ }`
	`241`	`+ }`
`208`	`242`	`}`
`209`	`243`	`})`
`210`	`244`
`@@ -260,6 +294,10 @@ object DeployPages : BuildType({`
`260`	`294`	`git push origin gh-pages`
`261`	`295`	`""".trimIndent()`
`262`	`296`	`formatStderrAsError = true`
	`297`	`+ dockerImage = "%DockerContainer%:%DockerVersion%"`
	`298`	`+ dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows`
	`299`	`+ dockerRunParameters = """--cpus=4 --memory=16g"""`
	`300`	`+ dockerPull = true`
`263`	`301`	`}`
`264`	`302`	`}`
`265`	`303`
`@@ -276,8 +314,12 @@ object DeployPages : BuildType({`
`276`	`314`	`}`
`277`	`315`	`}`
`278`	`316`
`279`		`- requirements {`
`280`		`- equals("env.OS", "Windows_NT")`
	`317`	`+ features {`
	`318`	`+ dockerSupport {`
	`319`	`+ loginToRegistry = on {`
	`320`	`+ dockerRegistryId = "PROJECT_EXT_134"`
	`321`	`+ }`
	`322`	`+ }`
`281`	`323`	`}`
`282`	`324`	`})`
`283`	`325`
Original file line number	Diff line number	Diff line change
`@@ -34,7 +34,7 @@ object UnitTestsTemplate : Template({`
`34`	`34`	`pixi run --environment default --frozen unittests`
`35`	`35`	`""".trimIndent()`
`36`	`36`	`formatStderrAsError = true`
`37`		`- dockerImage = "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1"`
	`37`	`+ dockerImage = "%DockerContainer%:%DockerVersion%"`
`38`	`38`	`dockerImagePlatform = ScriptBuildStep.ImagePlatform.Windows`
`39`	`39`	`dockerRunParameters = """--cpus=8 --memory=32g"""`
`40`	`40`	`dockerPull = true`
`@@ -58,7 +58,7 @@ object UnitTestsTemplate : Template({`
`58`	`58`	`""".trimIndent()`
`59`	`59`	`}`
`60`	`60`	`param("plugin.docker.imagePlatform", "windows")`
`61`		`- param("plugin.docker.imageId", "containers.deltares.nl/hydrology_product_line_imod/windows-pixi:v0.26.1")`
	`61`	`+ param("plugin.docker.imageId", "%DockerContainer%:%DockerVersion%")`
`62`	`62`	`param("plugin.docker.pull.enabled", "true")`
`63`	`63`	`}`
`64`	`64`	`}`