{GoogleSecret:SecretName:SecretVersion} Syntax (#12)

Author: neotrow

CHANGELOG.md

@@ -7,6 +7,10 @@ and adheres to a project-specific [Versioning](/README.md).
 
 ## [Unreleased]
 
+### Added
+
+Added support for the `{GoogleSecret:SecretName:SecretVersion}` syntax to override the default values in the `appsettings.json` or `appsettings.{Environment}.json` files.
+
 ## [1.2.0] - 2023-06-29
 
 ### Changed

GoogleSecrets/GoogleSecretsExtensions.cs

@@ -28,7 +28,7 @@ public static IConfigurationBuilder AddGoogleSecrets(this IConfigurationBuilder
                 throw new ArgumentNullException(nameof(googleSecretsOptions.ProjectName));
             }
 
-            configuration.Add(new GoogleSecretsSource(googleSecretsOptions));
+            configuration.Add(new GoogleSecretsSource(googleSecretsOptions, configuration.Build()));
 
             return configuration;
         }

GoogleSecrets/GoogleSecretsProvider.cs

@@ -3,6 +3,7 @@
     using System;
     using System.Collections.Generic;
     using System.Linq;
+    using System.Net.WebSockets;
     using System.Threading.Tasks;
     using Google.Api.Gax;
     using Google.Api.Gax.ResourceNames;
@@ -17,15 +18,20 @@
     public class GoogleSecretsProvider : ConfigurationProvider
     {
         private readonly ILogger<GoogleSecretsProvider> logger;
+        private readonly IConfigurationRoot existingConfiguration;
+        private readonly Dictionary<string, string> secretCache;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="GoogleSecretsProvider"/> class.
         /// </summary>
         /// <param name="source">The source.</param>
-        public GoogleSecretsProvider(GoogleSecretsSource source)
+        /// <param name="existingConfiguration">The configuration builder.</param>
+        public GoogleSecretsProvider(GoogleSecretsSource source, IConfigurationRoot existingConfiguration)
         {
             this.Source = source;
             this.logger = LoggerFactory.Create(b => b.AddConsole()).CreateLogger<GoogleSecretsProvider>();
+            this.existingConfiguration = existingConfiguration;
+            this.secretCache = new Dictionary<string, string>();
         }
 
         /// <summary>
@@ -56,41 +62,84 @@ public override void Load()
                 {
                     foreach (Secret secret in response)
                     {
-                        if (!this.Source.FilterFn(secret))
-                        {
-                            continue;
-                        }
-
-                        string version = "latest";
-                        var secretId = secret.SecretName.SecretId;
-
-                        if (this.Source.VersionDictionary?.ContainsKey(secretId) == true)
-                        {
-                            version = this.Source.VersionDictionary[secretId];
-                        }
-
                         try
                         {
-                            string versionName = $"{secret.Name}/versions/{version}";
-                            var value = secretManagerServiceClient.AccessSecretVersion(versionName);
-                            var secretValue = value.Payload.Data.ToStringUtf8();
+                            if (!this.Source.FilterFn(secret))
+                            {
+                                continue;
+                            }
 
-                            this.Set(this.Source.MapFn(secret), secretValue);
-                            this.logger.LogInformation($"Successfully loaded secret {secret.SecretName.SecretId}");
+                            ScanExistingConfiguration(secretManagerServiceClient, secret);
+                            ApplyMapFn(secretManagerServiceClient, secret);
                         }
                         catch (Exception e)
                         {
                             this.logger.LogWarning(e, $"Skipping secret {secret.SecretName.SecretId}");
                         }
-
-
                     }
                 }
             }
             catch (Exception e)
             {
                 this.logger.LogError(e, "Unhandeled Exception");
             }
+
+            void SetSecretValue(SecretManagerServiceClient secretManagerServiceClient, Secret secret, string key, string version)
+            {
+                string versionName = $"{secret.Name}/versions/{version}";
+
+                if (secretCache.TryGetValue(versionName, out var cachedValue))
+                {
+                    this.Set(key, cachedValue);
+                    this.logger.LogDebug($"Using cached value for secret {secret.SecretName.SecretId} Key: {key} Version: {version}");
+                }
+                else
+                {
+                    var value = secretManagerServiceClient.AccessSecretVersion(versionName);
+                    var secretValue = value.Payload.Data.ToStringUtf8();
+                    secretCache[versionName] = secretValue;
+                    this.Set(key, secretValue);
+                }
+
+                this.logger.LogInformation($"Successfully loaded secret {secret.SecretName.SecretId} into configuration. Key: {key} Version: {version}");
+            }
+
+            void ScanExistingConfiguration(SecretManagerServiceClient secretManagerServiceClient, Secret secret)
+            {
+                var existingKeyValues = this.existingConfiguration.AsEnumerable();
+                // value will be in the format of {GoogleSecret:SecretName} or {GoogleSecret:SecretName:Version}
+                var keyValuesToReplace = existingKeyValues.Where(x => x.Value?.StartsWith($"{{GoogleSecret:{secret.SecretName.SecretId}") == true);
+
+                foreach (var keyValue in keyValuesToReplace)
+                {
+                    var replaceParams = keyValue.Value.Split(':');
+                    string version = "latest";
+                    if (replaceParams.Length >= 3)
+                    {
+                        version = replaceParams[2];
+                    }
+
+                    SetSecretValue(secretManagerServiceClient, secret, keyValue.Key, version);
+                }
+            }
+
+            void ApplyMapFn(SecretManagerServiceClient secretManagerServiceClient, Secret secret)
+            {
+                if (this.Source.MapFn == null)
+                {
+                    return;
+                }
+
+                string version = "latest";
+                var secretId = secret.SecretName.SecretId;
+
+                if (this.Source.VersionDictionary?.ContainsKey(secretId) == true)
+                {
+                    version = this.Source.VersionDictionary[secretId];
+                }
+
+                SetSecretValue(secretManagerServiceClient, secret, this.Source.MapFn(secret), version);
+            }
         }
     }
 }

GoogleSecrets/GoogleSecretsSource.cs

@@ -13,11 +13,14 @@
     /// <seealso cref="Microsoft.Extensions.Configuration.IConfigurationSource" />
     public class GoogleSecretsSource : IConfigurationSource
     {
+        private readonly IConfigurationRoot existingConfig;
+
         /// <summary>
         /// Initializes a new instance of the <see cref="GoogleSecretsSource"/> class.
         /// </summary>
         /// <param name="options">The options.</param>
-        public GoogleSecretsSource(GoogleSecretsOptions options)
+        /// <param name="existingConfig">The existing configuration.</param>
+        public GoogleSecretsSource(GoogleSecretsOptions options, IConfigurationRoot existingConfig)
         {
             _ = options ?? throw new ArgumentNullException(nameof(options));
 
@@ -26,6 +29,7 @@ public GoogleSecretsSource(GoogleSecretsOptions options)
             this.ProjectName = options.ProjectName;
             this.Filter = options.Filter;
             this.VersionDictionary = options.VersionDictionary;
+            this.existingConfig = existingConfig;
         }
 
         /// <summary>
@@ -62,7 +66,7 @@ public GoogleSecretsSource(GoogleSecretsOptions options)
         /// </returns>
         public IConfigurationProvider Build(IConfigurationBuilder builder)
         {
-            return new GoogleSecretsProvider(this);
+            return new GoogleSecretsProvider(this, existingConfig);
         }
     }
 }

README.md

@@ -38,6 +38,10 @@ public static IHostBuilder CreateHostBuilder(string[] args)
 
 ## Overriding default appsettings.json values
 
+To override the default `appsettings.json` values you have to options which you can use together or separately:
+
+### Use the `MapFn` to map the secret to the correct path
+
 You can override existing `appsettings.json` values by making sure the `MapFn` maps it to the correct path.
 For example if we have the following `appsettings.json`:
 
@@ -51,6 +55,22 @@ For example if we have the following `appsettings.json`:
 
 We can override it by making sure that the `MapFn` maps it to `Secrets:MyGoogleSecret`. With the default `MapFn` the google secret would have to be called `Secrets__MyGoogleSecret`.
 
+### Use the {GoogleSecret:SecretName:SecretVersion} syntax
+
+In the `appsettings.json` you can use the `{GoogleSecret:SecretName:SecretVersion}` syntax to override the default values. For example:
+
+```json
+{
+  "Secrets": {
+    "MyGoogleSecret": "{GoogleSecret:MyGoogleSecret:latest}"
+  }
+}
+```
+
+will override the `MyGoogleSecret` with the latest version of the secret `MyGoogleSecret`.
+
+**Note: The Version is optional and can be omitted. If omitted the latest version will be taken.**
+
 ## Authentication
 
 ### Inside GCP