Skip to content

Commit ae4c16e

Browse files
mbt1mbt1
authored
Merge pull request #106 from tSQLt-org/tSQLtFacadeBuild
Days since last release 258
2 parents ffe4311 + 464d0f8 commit ae4c16e

106 files changed

Lines changed: 5613 additions & 1618 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/aks_build_and_test.yml‎

Lines changed: 492 additions & 0 deletions
Large diffs are not rendered by default.

‎.github/workflows/aks_build_container_images.yml‎

Lines changed: 294 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,294 @@
1+
# This workflow creates the ACR images for the different supported SQL Server versions.
2+
3+
name: Build MSSQL Images
4+
5+
on:
6+
# schedule:
7+
# # * is a special character in YAML so you have to quote this string
8+
# # Run every sunday at 6:20 AM
9+
# - cron: '20 6 * * 0'
10+
workflow_dispatch:
11+
inputs:
12+
projectName:
13+
description: 'Project Name'
14+
required: true
15+
default: 'test_20210224'
16+
machineName:
17+
description: 'Machine Name, cannot be more than 15 characters long, be entirely numeric, or contain the following characters: ` ~ ! @ # $ % ^ & * ( ) = + _ [ ] { } \ | ; : . singlequote " , < > / ?.'
18+
required: true
19+
default: 'w2019c1'
20+
msSqlVersions:
21+
description: 'names of the k8s deployment files in JSON array format (e.g. ["windows-2008r2","windows-2012","windows-2014","windows-2016","windows-2017","windows-2019"])'
22+
required: true
23+
default: '["windows-2014","windows-2016","windows-2017","windows-2019"]'
24+
debugVM:
25+
description: 'set to "true" to retain the VM for debugging purposes'
26+
required: false
27+
default: 'false'
28+
29+
30+
jobs:
31+
32+
# 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖
33+
create-vm:
34+
name: Create The VM
35+
runs-on: windows-latest
36+
# if: ${{ false }}
37+
38+
env:
39+
PROJECT_NAME: ${{ github.event.inputs.projectName }}
40+
MACHINE_NAME: ${{ github.event.inputs.machineName }}
41+
MS_SQL_VERSIONS: ${{ github.event.inputs.msSqlVersions }}
42+
AZ_SERVICE_PRINCIPAL_CREDENTIALS: ${{ secrets[format('AZ_SP_CRED_{0}', github.event.inputs.projectName)] }}
43+
44+
steps:
45+
- name: Checkout self
46+
uses: actions/checkout@v2
47+
with:
48+
path: cicd
49+
50+
- name: create variables
51+
id: create-vars
52+
shell: pwsh
53+
run: |
54+
function Get-MD5HashOfString($string) {
55+
$stringAsStream = [System.IO.MemoryStream]::new();
56+
$writer = [System.IO.StreamWriter]::new($stringAsStream);
57+
$writer.write($string);
58+
$writer.Flush();
59+
$stringAsStream.Position = 0;
60+
$hashedString = (Get-FileHash -InputStream $stringAsStream).Hash;
61+
return [String]$hashedString;
62+
}
63+
64+
$projectNameHash = (Get-MD5HashOfString($env:PROJECT_NAME)).Substring(0,10);
65+
$machineName = "$env:MACHINE_NAME";
66+
$machineRgName = "rg_$machineName";
67+
$azSecretsManagerName = "sm-" + $projectNameHash;
68+
$containerRegistryURL = "crn" + $projectNameHash + ".azurecr.io";
69+
$repoURL = "${{ github.SERVER_URL }}/${{ github.REPOSITORY }}.git";
70+
$commitId = "${{ github.SHA }}" ;
71+
$msSqlVersionForMatrix = '{"windows-version":'+$env:MS_SQL_VERSIONS+'}';
72+
73+
74+
Write-Host "✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ";
75+
Write-Host "";
76+
Write-Host ("projectName value: {0}" -f $env:PROJECT_NAME);
77+
Write-Host ("machineName value: {0}" -f $machineName);
78+
Write-Host ("machineRgName value: {0}" -f $machineRgName);
79+
Write-Host ("msSqlVersionForMatrix: {0}" -f "$msSqlVersionForMatrix");
80+
Write-Host ("azSecretsManagerName: {0}" -f "$azSecretsManagerName");
81+
Write-Host ("containerRegistryURL: {0}" -f "$containerRegistryURL");
82+
Write-Host ("repoURL: {0}" -f "$repoURL");
83+
Write-Host ("commitId: {0}" -f "$commitId");
84+
Write-Host "";
85+
Write-Host "✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ";
86+
87+
Write-Host ('::set-output name=azsecretsmanagername::'+$azSecretsManagerName);
88+
Write-Host ('::set-output name=containerregistryurl::'+$containerRegistryURL);
89+
Write-Host ('::set-output name=repoURL::'+$repoURL);
90+
Write-Host ('::set-output name=machineName::'+$machineName);
91+
Write-Host ('::set-output name=machineRgName::'+$machineRgName);
92+
Write-Host ('::set-output name=commitId::'+$commitId);
93+
Write-Host ("::set-output name=msSqlVersionForMatrix::{0}" -f "$msSqlVersionForMatrix");
94+
95+
- name: decode az sp cred
96+
id: cred-decode
97+
shell: pwsh
98+
run: |
99+
$decodedCreds = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String("${{ env.AZ_SERVICE_PRINCIPAL_CREDENTIALS}}"))
100+
Write-Host ('::set-output name=az_sp_creds::'+$decodedCreds);
101+
102+
# documentation: https://github.com/azure/login#configure-azure-credentials
103+
- name: login via az module
104+
uses: azure/login@v1
105+
with:
106+
creds: ${{ steps.cred-decode.outputs.az_sp_creds }}
107+
enable-AzPSSession: true
108+
109+
# documentation: https://github.com/Azure/get-keyvault-secrets
110+
- name: get azure secrets
111+
id: azure-secrets
112+
uses: Azure/get-keyvault-secrets@v1.0
113+
env:
114+
ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true' # Note that this task can be replaced with a similar pattern as setting the namespace to the env variables (above), but is also not secure.
115+
with:
116+
keyvault: ${{ steps.create-vars.outputs.azsecretsmanagername }}
117+
secrets: 'azResourceGroupName' # comma separated list of secret keys that need to be fetched from the Key Vault
118+
119+
- name: run azure powershell script
120+
id: buildMachine-ps
121+
uses: azure/powershell@v1
122+
with:
123+
azpsversion: 'latest'
124+
errorActionPreference: 'continue'
125+
inlineScript: |
126+
$Parameters = @{
127+
projectName = "$env:PROJECT_NAME";
128+
azSecretsManagerName = "${{ steps.create-vars.outputs.azsecretsmanagername }}";
129+
azResourceGroupName = "${{ steps.azure-secrets.outputs.azResourceGroupName }}";
130+
machineRgName = "${{ steps.create-vars.outputs.machineRgName }}";
131+
repoURL = "${{ steps.create-vars.outputs.repoURL }}";
132+
commitId = "${{ steps.create-vars.outputs.commitId }}";
133+
machineName = "${{ steps.create-vars.outputs.machineName }}";
134+
debugOn = $true;
135+
};
136+
./cicd/envSetup/createContainerBuildMachine.ps1 @Parameters;
137+
138+
outputs:
139+
mssqlversion: ${{ steps.create-vars.outputs.msSqlVersionForMatrix }}
140+
containerregistryurl: ${{ steps.create-vars.outputs.containerregistryurl }}
141+
azsecretsmanagername: ${{ steps.create-vars.outputs.azSecretsManagerName }}
142+
machinergname: ${{ steps.create-vars.outputs.machineRgName }}
143+
machinename: ${{ steps.create-vars.outputs.machineName }}
144+
145+
# 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖
146+
create-images:
147+
name: create and publish the ACR images
148+
needs: create-vm
149+
150+
runs-on: windows-latest
151+
152+
env:
153+
AZ_CONTAINER_REGISTRY_URL: ${{ needs.create-vm.outputs.containerregistryurl }}
154+
AZ_SECRETS_MANAGER_NAME: ${{ needs.create-vm.outputs.azsecretsmanagername }}
155+
MACHINE_NAME: ${{ needs.create-vm.outputs.machinename }}
156+
MACHINE_RG_NAME: ${{ needs.create-vm.outputs.machinergname }}
157+
AZ_SERVICE_PRINCIPAL_CREDENTIALS: ${{ secrets[format('AZ_SP_CRED_{0}', github.event.inputs.projectName)] }}
158+
159+
strategy:
160+
matrix: ${{fromJson(needs.create-vm.outputs.mssqlversion)}}
161+
max-parallel: 1
162+
163+
steps:
164+
165+
- name: Checkout self
166+
uses: actions/checkout@v2
167+
with:
168+
path: cicd
169+
170+
- name: build and push docker image
171+
shell: pwsh
172+
run: |
173+
$mssqlversion = '${{ matrix.mssqlversion }}';
174+
175+
Write-Host "✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ";
176+
Write-Host "";
177+
Write-Host ("mssqlversion: {0}" -f "$mssqlversion");
178+
Write-Host ("azContainerRegistryURL: {0}" -f "$env:AZ_CONTAINER_REGISTRY_URL");
179+
Write-Host ("azSecretsManagerName: {0}" -f "$env:AZ_SECRETS_MANAGER_NAME");
180+
Write-Host ("machineName: {0}" -f "$env:MACHINE_NAME");
181+
Write-Host ("machineRGName: {0}" -f "$env:MACHINE_RG_NAME");
182+
Write-Host "";
183+
Write-Host "✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ✨ ";
184+
185+
- name: decode az sp cred
186+
id: cred-decode
187+
shell: pwsh
188+
run: |
189+
$decodedCreds = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String("${{ env.AZ_SERVICE_PRINCIPAL_CREDENTIALS}}"));
190+
Write-Host ('::set-output name=az_sp_creds::'+$decodedCreds);
191+
192+
# documentation: https://github.com/azure/login#configure-azure-credentials
193+
- name: login via az module
194+
uses: azure/login@v1
195+
with:
196+
creds: ${{ steps.cred-decode.outputs.az_sp_creds }}
197+
enable-AzPSSession: true
198+
199+
- name: build and push image script
200+
id: build-and-push-image
201+
uses: azure/powershell@v1
202+
with:
203+
azpsversion: 'latest'
204+
errorActionPreference: 'continue'
205+
inlineScript: |
206+
$builImageScriptPath = 'cicd\envSetup\buildAndPushImage.ps1';
207+
Invoke-AzVMRunCommand -ResourceGroupName $env:MACHINE_RG_NAME -VMName $env:MACHINE_NAME -CommandId 'RunPowerShellScript' -ScriptPath $builImageScriptPath -Parameter @{mssqlVersion = "${{ matrix.mssqlversion }}"; acrURL = "$env:AZ_CONTAINER_REGISTRY_URL"; azSpCrBase64 = "${{ env.AZ_SERVICE_PRINCIPAL_CREDENTIALS}}"; debugOnString = "$debugOn"}
208+
209+
# 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖
210+
prune-untagged-images:
211+
name: prune images
212+
needs: [create-vm, create-images]
213+
if: ${{ always() }}
214+
215+
runs-on: windows-latest
216+
217+
env:
218+
AZ_CONTAINER_REGISTRY_URL: ${{ needs.create-vm.outputs.containerregistryurl }}
219+
AZ_SERVICE_PRINCIPAL_CREDENTIALS: ${{ secrets[format('AZ_SP_CRED_{0}', github.event.inputs.projectName)] }}
220+
221+
steps:
222+
- name: decode az sp cred
223+
id: cred-decode
224+
shell: pwsh
225+
run: |
226+
$decodedCreds = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String("${{ env.AZ_SERVICE_PRINCIPAL_CREDENTIALS}}"))
227+
Write-Host ('::set-output name=az_sp_creds::'+$decodedCreds);
228+
229+
# documentation: https://github.com/azure/login#configure-azure-credentials
230+
- name: login via az module
231+
uses: azure/login@v1
232+
with:
233+
creds: ${{ steps.cred-decode.outputs.az_sp_creds }}
234+
enable-AzPSSession: true
235+
236+
- name: drop image script
237+
id: dropBuildMachine-ps
238+
uses: azure/powershell@v1
239+
with:
240+
azpsversion: 'latest'
241+
errorActionPreference: 'continue'
242+
inlineScript: |
243+
# Documented here: https://docs.microsoft.com/en-us/azure/container-registry/container-registry-delete#delete-all-untagged-images
244+
245+
$registry = "$env:AZ_CONTAINER_REGISTRY_URL";
246+
$repository = "windows-mssql"; # https://github.com/distribution/distribution/blob/main/docs/spec/api.md#overview, must adhere to: [a-z0-9]+(?:[._-][a-z0-9]+)*
247+
248+
az acr repository show-manifests --name $registry --repository $repository --query "[?tags[0]==null].digest" -o tsv | %{ az acr repository delete --name $registry --image $repository@$_ --yes }
249+
250+
251+
# 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖 💖
252+
drop-vm:
253+
name: drop the VM
254+
needs: [create-vm, create-images]
255+
if: ${{ always() }}
256+
257+
runs-on: windows-latest
258+
259+
env:
260+
AZ_SERVICE_PRINCIPAL_CREDENTIALS: ${{ secrets[format('AZ_SP_CRED_{0}', github.event.inputs.projectName)] }}
261+
MACHINE_RG_NAME: ${{ needs.create-vm.outputs.machinergname }}
262+
263+
steps:
264+
- name: check debug VM
265+
shell: pwsh
266+
if: ${{ github.event.inputs.debugVM == 'true' }}
267+
run: |
268+
Write-Host ("::error::✨ ✨ ✨TURN OFF THE VM WHEN YOU ARE DONE!✨ ✨ ✨");
269+
exit 1;
270+
271+
- name: decode az sp cred
272+
id: cred-decode
273+
shell: pwsh
274+
run: |
275+
$decodedCreds = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String("${{ env.AZ_SERVICE_PRINCIPAL_CREDENTIALS}}"))
276+
Write-Host ('::set-output name=az_sp_creds::'+$decodedCreds);
277+
278+
# documentation: https://github.com/azure/login#configure-azure-credentials
279+
- name: login via az module
280+
uses: azure/login@v1
281+
with:
282+
creds: ${{ steps.cred-decode.outputs.az_sp_creds }}
283+
enable-AzPSSession: true
284+
285+
- name: drop image build machine resource group script
286+
id: dropBuildMachine-ps
287+
uses: azure/powershell@v1
288+
with:
289+
azpsversion: 'latest'
290+
errorActionPreference: 'continue'
291+
inlineScript: |
292+
Remove-AzResourceGroup -Name "$env:MACHINE_RG_NAME" -Force ;
293+
294+

‎.github/workflows/aks_cicd_environment_setup.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# This is the workflow to setup the environment for Data Platform CI/CD pipelines
2+
#
3+
# AZURE: Before this workflow will run successfully you must do the following
4+
# 1. Run manualPrep.ps1 in the Azure Portal and follow the instructions to set up the AZ_SP_CRED_<projectName> secret in GitHub (Settings > Secrets > "New repository secret").
5+
# 2. Set up the SSH_PASSPHRASE secret in Settings > Secrets > "New repository secret"
6+
7+
name: Set Up CI/CD Environment
8+
9+
on:
10+
workflow_dispatch:
11+
inputs:
12+
projectName:
13+
description: 'Project Name'
14+
required: true
15+
default: 'db_cicd_project'
16+
linuxNodePoolDefaultVMSize:
17+
description: 'Linux Node Pool Default VM Size'
18+
required: true
19+
default: 'Standard_D2_v2'
20+
windowsNodePoolDefaultVMSize:
21+
description: 'Windows Node Pool Default VM Size'
22+
required: true
23+
default: 'Standard_D3_v2'
24+
kubernetesVersion:
25+
description: 'Kubernetes Version'
26+
required: true
27+
default: '1.19.6'
28+
29+
jobs:
30+
build:
31+
name: Setup Azure Environment
32+
runs-on: ubuntu-latest
33+
34+
env:
35+
PROJECT_NAME: ${{ github.event.inputs.projectName }}
36+
LINUX_NODEPOOL_DEFAULT_VM_SIZE: ${{ github.event.inputs.linuxNodePoolDefaultVMSize }}
37+
WINDOWS_NODEPOOL_DEFAULT_VM_SIZE: ${{ github.event.inputs.windowsNodePoolDefaultVMSize }}
38+
KUBERNETES_VERSION: ${{ github.event.inputs.kubernetesVersion }}
39+
AZ_SERVICE_PRINCIPAL_CREDENTIALS: ${{ secrets[format('AZ_SP_CRED_{0}', github.event.inputs.projectName)] }}
40+
SSH_PASSPHRASE: ${{ secrets.SSH_PASSPHRASE }}
41+
42+
steps:
43+
- uses: actions/checkout@v2
44+
45+
- name: decode az sp cred
46+
id: cred-decode
47+
shell: pwsh
48+
run: |
49+
$decodedCreds = [System.Text.Encoding]::Unicode.GetString([System.Convert]::FromBase64String("${{ env.AZ_SERVICE_PRINCIPAL_CREDENTIALS}}"))
50+
Write-Host ('::set-output name=az_sp_creds::'+$decodedCreds);
51+
52+
# documentation: https://github.com/azure/login#configure-azure-credentials
53+
# TODO: set up a service principal which has permission only on the resource group and associated resources it creates
54+
# MDP: This task can be replaced with Connect-AzAccount via service principal (https://docs.microsoft.com/en-us/powershell/azure/authenticate-azureps?view=azps-5.3.0). However for most pipelines, which would use azure powershell more than once, this is an anti-pattern.
55+
# TODO: measure difference in performance between using this task and Connect-AzAccount in the next task
56+
- name: login via az module
57+
uses: azure/login@v1
58+
with:
59+
creds: ${{ steps.cred-decode.outputs.az_sp_creds }}
60+
enable-azpssession: true
61+
62+
# documentation: https://github.com/marketplace/actions/azure-powershell-action
63+
- name: run azure powershell script
64+
uses: azure/powershell@v1
65+
with:
66+
azpsversion: 'latest'
67+
errorActionPreference: 'continue'
68+
inlineScript: |
69+
$Parameters = @{
70+
projectName = "$env:PROJECT_NAME";
71+
azServicePrincipalCredentials = '${{ steps.cred-decode.outputs.az_sp_creds }}';
72+
sshPassphrase = "$env:SSH_PASSPHRASE";
73+
linuxNodePoolDefaultVMSize = "$env:LINUX_NODEPOOL_DEFAULT_VM_SIZE"
74+
windowsNodePoolDefaultVMSize = "$env:WINDOWS_NODEPOOL_DEFAULT_VM_SIZE"
75+
kubernetesVersion = "$env:KUBERNETES_VERSION"
76+
debugOn = $false;
77+
};
78+
./envSetup/setup.ps1 @Parameters;

0 commit comments

Comments
 (0)