Merge pull request #6 from bugout-dev/parameters-go

Go version of parameters script

Author: kompotkot

README.md

@@ -20,10 +20,11 @@ except they explicitly `export` the environment variables. We source these envir
 run the development versions of our applications.
 
 It can sometimes be difficult to understand:
-1. Whether all the environment variables we *expect* to be defined in production actually have been.
+
+1. Whether all the environment variables we _expect_ to be defined in production actually have been.
 2. What the particular value of a production environment actually is.
 3. What the differences are between our expectations and the actual environment variables in a running
-application process.
+   application process.
 
 We are building and maintaining `checkenv` to make it easier for us to diagnose and fix issues with
 application configuration via environment variables. We stand in solidarity with anyone else who
@@ -40,3 +41,24 @@ binary which supports your needs.
 
 There is currently no need to support runtime plugins. Since doing so would make this program a lot
 more complicated, we have decided to forego runtime plugin functionality for now.
+
+## Usage
+
+```bash
+./checkenv plugins
+```
+
+Available plugins:
+
+-   env - Provides the environment variables defined in the checkenv process.
+-   file - Provides the environment variables defined in the env file with the given path.
+-   proc - Provides the environment variables set for the process with the given pid.
+-   aws_ssm - Provides environment variables defined in AWS Systems Manager Parameter Store.
+
+### aws_ssm plugin
+
+In order to fetch parameters with tags `Product` = `test` and `Node` = `true` with `export ` prefix execute following command
+
+```bash
+./checkenv show -export aws_ssm+Product:test,Node:true
+```

aws_ssm.go

@@ -0,0 +1,38 @@
+// checkenv plugin that provides environment variables defined in AWS System Manager Parameter Store.
+
+package main
+
+import (
+	"context"
+
+	"github.com/bugout-dev/checkenv/aws_ssm"
+)
+
+func AWSSystemsManagerParameterStoreProvider(filter string) (map[string]string, error) {
+	environment := make(map[string]string)
+
+	// Convert string of tags for filter to key:value structure
+	filterTags := aws_ssm.ParseFilterTags(filter)
+
+	ctx := context.Background()
+
+	api := aws_ssm.InitAWSClient(ctx)
+
+	keys := aws_ssm.FetchKeysOfParameters(ctx, api, filterTags)
+
+	// Split slice of parameter keys to chunks by 10 (max len allowed by AWS)
+	// and fetch values for required parameters
+	keyChunks := aws_ssm.GenerateChunks(keys, 10)
+	parameters := aws_ssm.FetchParameters(ctx, api, keyChunks)
+
+	for _, parameter := range parameters {
+		environment[parameter.Name] = parameter.Value
+	}
+
+	return environment, nil
+}
+
+func init() {
+	helpString := "Provides environment variables defined in AWS Systems Manager Parameter Store."
+	RegisterPlugin("aws_ssm", helpString, noop, AWSSystemsManagerParameterStoreProvider)
+}

aws_ssm/aws_ssm.go

@@ -0,0 +1,56 @@
+/*
+Based on: https://github.com/awsdocs/aws-doc-sdk-examples/blob/main/gov2/ssm/GetParameter/GetParameterv2.go
+*/
+package aws_ssm
+
+import (
+	"context"
+	"log"
+
+	"github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/ssm"
+)
+
+// AWSSystemsManagerParameterStoreAPI defines the interface
+// for the GetParameters and DescribeParameters function.
+// We use this interface to test the function using a mocked service
+type AWSSystemsManagerParameterStoreAPI interface {
+	GetParameters(
+		ctx context.Context,
+		params *ssm.GetParametersInput,
+		optFns ...func(*ssm.Options),
+	) (*ssm.GetParametersOutput, error)
+
+	DescribeParameters(
+		ctx context.Context,
+		params *ssm.DescribeParametersInput,
+		optFns ...func(*ssm.Options),
+	) (*ssm.DescribeParametersOutput, error)
+}
+
+// ExecGetParameters and ExecDescribeParameters retrieves an AWS Systems Manager string parameter
+// Inputs:
+// 		c: is the context of the method call, which includes the AWS Region
+// 		api: is the interface that defines the method call
+// 		input: defines the input arguments to the service call
+// Output:
+// 		If success, a GetParametersOutput object containing the result of the service call and nil
+// 		Otherwise, nil and an error from the call to GetParameters
+func ExecGetParameters(c context.Context, api AWSSystemsManagerParameterStoreAPI, input *ssm.GetParametersInput) (*ssm.GetParametersOutput, error) {
+	return api.GetParameters(c, input)
+}
+
+func ExecDescribeParameters(c context.Context, api AWSSystemsManagerParameterStoreAPI, input *ssm.DescribeParametersInput) (*ssm.DescribeParametersOutput, error) {
+	return api.DescribeParameters(c, input)
+}
+
+// Load the Shared AWS Configuration (~/.aws/config)
+func InitAWSClient(ctx context.Context) *ssm.Client {
+	cfg, err := config.LoadDefaultConfig(ctx)
+	if err != nil {
+		log.Fatalln("Failed loading AWS Configuration", err)
+	}
+	client := ssm.NewFromConfig(cfg)
+
+	return client
+}

aws_ssm/data.go

@@ -0,0 +1,13 @@
+package aws_ssm
+
+// Parameter structure for storing final result from AWS SSM
+type Parameter struct {
+	Name  string
+	Value string
+}
+
+// Tags for filter defined by user
+type FilterTag struct {
+	Name  string
+	Value string
+}

aws_ssm/data_test.json

@@ -0,0 +1,44 @@
+[
+    {
+        "Name": "/wrong/dev/y1",
+        "Value": "w1",
+        "Tags": [
+            {
+                "Product": "wrong"
+            }
+        ]
+    },
+    {
+        "Name": "/test/dev/t1",
+        "Value": "q1",
+        "Tags": [
+            {
+                "Product": "test"
+            }
+        ]
+    },
+    {
+        "Name": "/test/dev/t2",
+        "Value": "q2",
+        "Tags": [
+            {
+                "Product": "test",
+                "Application": "dev"
+            }
+        ]
+    },
+    {
+        "Name": "/test/dev/t3",
+        "Value": "q3",
+        "Tags": []
+    },
+    {
+        "Name": "/test/dev/t4",
+        "Value": "q4",
+        "Tags": [
+            {
+                "Product": "wrong"
+            }
+        ]
+    }
+]

aws_ssm/parameters.go

@@ -0,0 +1,141 @@
+package aws_ssm
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/aws/aws-sdk-go-v2/service/ssm"
+	"github.com/aws/aws-sdk-go-v2/service/ssm/types"
+)
+
+// Fetch values for parameters
+// Inputs:
+//		chunks: list of lists with parameter key values
+func FetchParameters(ctx context.Context, api AWSSystemsManagerParameterStoreAPI, chunks [][]string) []Parameter {
+	var parameters []Parameter
+
+	for _, chunk := range chunks {
+		getInput := &ssm.GetParametersInput{
+			Names: chunk,
+		}
+		results, err := ExecGetParameters(ctx, api, getInput)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, p := range results.Parameters {
+			parameter := Parameter{
+				Name: *p.Name, Value: *p.Value,
+			}
+			parameters = append(parameters, parameter)
+		}
+	}
+	log.Println("Retrieved values for parameters")
+
+	return parameters
+}
+
+// Fetch list of parameter keys from AWS with defined filters
+func FetchKeysOfParameters(
+	ctx context.Context,
+	api AWSSystemsManagerParameterStoreAPI,
+	filterTags []FilterTag,
+) []string {
+	var parameters []string
+
+	// Set parameter filters
+	parameterFilters := []types.ParameterStringFilter{}
+	for _, ft := range filterTags {
+		filterKey := fmt.Sprintf("tag:%s", ft.Name)
+		parameterFilters = append(parameterFilters, types.ParameterStringFilter{
+			Key:    &filterKey,
+			Values: []string{ft.Value},
+		})
+	}
+	describeInput := &ssm.DescribeParametersInput{
+		MaxResults:       10,
+		ParameterFilters: parameterFilters,
+	}
+
+	// CHECKENV_AWS_FETCH_LOOP_LIMIT by default set to 10,
+	// it is allows to load 100 parameters from AWS and it is
+	// a limiter to prevent loading too many parameters without
+	// control during passing erroneous filters
+	var err error
+	var fetchLoopLimit int
+	fetchLoopLimitStr := os.Getenv("CHECKENV_AWS_FETCH_LOOP_LIMIT")
+	if fetchLoopLimitStr != "" {
+		fetchLoopLimit, err = strconv.Atoi(fetchLoopLimitStr)
+	}
+	if fetchLoopLimitStr == "" || err != nil {
+		fetchLoopLimit = 10
+	}
+
+	n := 0
+	for {
+		// Fetch list of parameter keys
+		results, err := ExecDescribeParameters(ctx, api, describeInput)
+		if err != nil {
+			log.Fatal(err)
+		}
+		for _, p := range results.Parameters {
+			parameters = append(parameters, *p.Name)
+		}
+
+		// If there are no more parameters break
+		if results.NextToken == nil {
+			break
+		}
+		describeInput.NextToken = results.NextToken
+
+		n++
+		if n >= fetchLoopLimit {
+			log.Fatal("To many iterations over DescribeParameters loop")
+		}
+	}
+	log.Printf("Retrieved %d parameters", len(parameters))
+
+	return parameters
+}
+
+// Split list of reports on nested lists
+func GenerateChunks(flatSlice []string, chunkSize int) [][]string {
+	if len(flatSlice) == 0 {
+		return [][]string{}
+	}
+
+	chunks := make([][]string, 0, len(flatSlice)/chunkSize+1)
+
+	for i, v := range flatSlice {
+		if i%chunkSize == 0 {
+			chunks = append(chunks, make([]string, 0, chunkSize))
+		}
+		chunks[len(chunks)-1] = append(chunks[len(chunks)-1], v)
+	}
+
+	return chunks
+}
+
+// ParseFilterTags convert string from user input to key value structure
+func ParseFilterTags(filterTagsStr string) []FilterTag {
+	var filterTags []FilterTag
+
+	filterTagsSlice := strings.Split(filterTagsStr, ",")
+	for _, t := range filterTagsSlice {
+		tagNameValue := strings.Split(t, ":")
+		if len(tagNameValue) != 2 || len(tagNameValue[0]) == 0 || len(tagNameValue[1]) == 0 {
+			log.Printf("Unable to parse tag name and value: %s", t)
+			continue
+		}
+		filterTags = append(filterTags, FilterTag{
+			Name:  tagNameValue[0],
+			Value: tagNameValue[1],
+		})
+	}
+
+	return filterTags
+}

aws_ssm/parameters_test.go

@@ -0,0 +1,50 @@
+package aws_ssm
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestGenerateChunks(t *testing.T) {
+	var cases = []struct {
+		flatSlice []string
+		chunkSIze int
+		expected  [][]string
+	}{
+		{[]string{}, 1, [][]string{}},
+		{[]string{}, 2, [][]string{}},
+		{[]string{"val-1", "val-2"}, 2, [][]string{{"val-1", "val-2"}}},
+		{[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 1, [][]string{{"val-1"}, {"val-2"}, {"val-3"}, {"val-4"}, {"val-5"}}},
+		{[]string{"val-1", "val-2", "val-3", "val-4", "val-5"}, 2, [][]string{{"val-1", "val-2"}, {"val-3", "val-4"}, {"val-5"}}},
+		{[]string{"val-1", "val-2", "val-3", "val-4", "val-5", "val-6"}, 3, [][]string{{"val-1", "val-2", "val-3"}, {"val-4", "val-5", "val-6"}}},
+	}
+	for _, c := range cases {
+		chunks := GenerateChunks(c.flatSlice, c.chunkSIze)
+		if !reflect.DeepEqual(chunks, c.expected) {
+			t.Logf("Value should be %s, but got %s", c.expected, chunks)
+			t.Fail()
+		}
+	}
+}
+
+func TestFilterTags(t *testing.T) {
+	var emptyFilterTags []FilterTag
+	var cases = []struct {
+		filterTagsStr string
+		expected      []FilterTag
+	}{
+		{"Product", emptyFilterTags},
+		{"Product:", emptyFilterTags},
+		{":test", emptyFilterTags},
+		{":", emptyFilterTags},
+		{"Product:test", []FilterTag{{Name: "Product", Value: "test"}}},
+		{"Product:test,Node:true", []FilterTag{{Name: "Product", Value: "test"}, {Name: "Node", Value: "true"}}},
+	}
+	for _, c := range cases {
+		filterTags := ParseFilterTags(c.filterTagsStr)
+		if !reflect.DeepEqual(filterTags, c.expected) {
+			t.Logf("Value should be %s, but got %s", c.expected, filterTags)
+			t.Fatal()
+		}
+	}
+}