Merge pull request #108 from mdelapenya/103-fix-metricbeat-tests

mdelapenya · web-flow · commit fee74639edc4 · 2020-04-24T17:00:34.000+02:00
Fix metricbeat tests
diff --git a/.ci/.e2e-tests.yaml b/.ci/.e2e-tests.yaml
@@ -2,5 +2,6 @@
 FEATURES:
   - "apache"
   - "helm"
+  - "metricbeat"
   - "mysql"
   - "redis"
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -25,6 +25,8 @@ pipeline {
   }
   parameters {
     choice(name: 'LOG_LEVEL', choices: ['INFO', 'DEBUG'], description: 'Log level to be used')
+    choice(name: 'QUERY_MAX_ATTEMPTS', choices: ['5', '10', '20'], description: 'Number of attempts to create the connection to Elasticsearch')
+    choice(name: 'RETRY_TIMEOUT', choices: ['3', '5', '7', '11'], description: 'Number of seconds between retry')
     string(name: 'STACK_VERSION', defaultValue: '7.6.0', description: 'SemVer version of the stack to be used.')
     string(name: 'METRICBEAT_VERSION', defaultValue: '7.6.0', description: 'SemVer version of the metricbeat to be used.')
     string(name: 'HELM_CHART_VERSION', defaultValue: '7.6.1', description: 'SemVer version of Helm chart to be used.')
@@ -51,6 +53,8 @@ pipeline {
         KIND_VERSION = "${params.KIND_VERSION.trim()}"
         KUBERNETES_VERSION = "${params.KUBERNETES_VERSION.trim()}"
         LOG_LEVEL = "${params.LOG_LEVEL.trim()}"
+        QUERY_MAX_ATTEMPTS = "${params.QUERY_MAX_ATTEMPTS.trim()}"
+        RETRY_TIMEOUT = "${params.RETRY_TIMEOUT.trim()}"
       }
       stages {
         stage('Checkout') {
@@ -183,6 +187,11 @@ def generateFunctionalTestStep(Map params = [:]){
       try {
         deleteDir()
         unstash 'build'
+        retry(3){
+          dir("${BASE_DIR}"){
+            sh script: """.ci/scripts/install-test-dependencies.sh "${GO_VERSION}" """, label: "Install test dependencies for ${feature}"
+          }
+        }
         dir("${BASE_DIR}"){
           sh script: """.ci/scripts/functional-test.sh "${GO_VERSION}" "${feature}" "${STACK_VERSION}" "${METRICBEAT_VERSION}" """, label: "Run functional tests for ${feature}"
         }
diff --git a/.ci/functionalTests.groovy b/.ci/functionalTests.groovy
@@ -107,6 +107,11 @@ pipeline {
       steps {
         deleteDir()
         unstash 'source'
+        retry(3){
+          dir("${BASE_DIR}"){
+            sh script: """.ci/scripts/install-test-dependencies.sh "${GO_VERSION}" """, label: "Install test dependencies for ${params.FEATURE}"
+          }
+        }
         dir(BASE_DIR){
           sh script: """.ci/scripts/functional-test.sh "${GO_VERSION}" "${params.FEATURE}" """, label: 'Run functional tests'
         }
diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh
@@ -19,15 +19,9 @@ TARGET_ARCH=${GOARCH:-amd64}
 # shellcheck disable=SC1091
 source .ci/scripts/install-go.sh "${GO_VERSION}"
 
-# shellcheck disable=SC1091
-source .ci/scripts/install-test-dependencies.sh
-
 # Build OP Binary
 GOOS=${TARGET_OS} GOARCH=${TARGET_ARCH} make -C e2e fetch-binary
 
-# Build runtime dependencies
-STACK_VERSION=${STACK_VERSION} make -C e2e run-elastic-stack
-
 # Sync integrations
 make -C e2e sync-integrations
 
diff --git a/.ci/scripts/install-test-dependencies.sh b/.ci/scripts/install-test-dependencies.sh
@@ -4,11 +4,17 @@ set -euxo pipefail
 # Install the dependencies using the install and test make goals.
 #
 # Parameters:
+#   - GO_VERSION - that's the Go version which will be used for running Go code.
 #   - HELM_VERSION - that's the Helm version which will be installed and enabled.
 #   - KIND_VERSION - that's the Kind version which will be installed and enabled.
 #   - KUBERNETES_VERSION - that's the Kubernetes version which will be installed and enabled.
 #
 
+GO_VERSION=${1:?GO_VERSION is not set}
+
+# shellcheck disable=SC1091
+source .ci/scripts/install-go.sh "${GO_VERSION}"
+
 MSG="parameter missing."
 HOME=${HOME:?$MSG}
 
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -102,12 +102,12 @@
             "type": "go",
             "request": "launch",
             "mode": "auto",
-            "program": "${workspaceFolder}/e2e",
+            "program": "${file}",
             "env": {
                 "GO111MODULE": "on",
                 "OP_LOG_LEVEL": "DEBUG",
             },
-            "args": ["godog", "-t", "mysql"]
+            "args": ["--godog.format", "pretty", "metricbeat"]
         }
     ]
 }
diff --git a/cli/config/compose/services/metricbeat/docker-compose.yml b/cli/config/compose/services/metricbeat/docker-compose.yml
@@ -3,6 +3,7 @@ services:
   metricbeat:
     command: [
       "metricbeat", "-e",
+      "-E", "logging.level=${logLevel}",
       "-E", "setup.ilm.rollover_alias=${indexName}",
       "-E", "output.elasticsearch.hosts=http://elasticsearch:9200",
       "-E", "output.elasticsearch.password=p4ssw0rd",
diff --git a/cli/config/compose/stacks/metricbeat/docker-compose.yml b/cli/config/compose/stacks/metricbeat/docker-compose.yml
@@ -12,11 +12,3 @@ services:
     image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion}"
     ports:
       - "9200:9200"
-  kibana:
-    environment:
-      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
-      - ELASTIC_USERNAME=elastic
-      - ELASTIC_PASSWORD=p4ssw0rd
-    image: "docker.elastic.co/kibana/kibana:${stackVersion}"
-    ports:
-      - "5601:5601"
diff --git a/cli/services/manager.go b/cli/services/manager.go
@@ -15,6 +15,7 @@ import (
 type ServiceManager interface {
 	AddServicesToCompose(stack string, composeNames []string, env map[string]string) error
 	RemoveServicesFromCompose(stack string, composeNames []string, env map[string]string) error
+	RunCommand(stack string, composeNames []string, composeArgs []string, env map[string]string) error
 	RunCompose(isStack bool, composeNames []string, env map[string]string) error
 	StopCompose(isStack bool, composeNames []string) error
 }
@@ -84,6 +85,11 @@ func (sm *DockerServiceManager) RemoveServicesFromCompose(stack string, composeN
 	return nil
 }
 
+// RunCommand executes a docker-compose command in a running a docker compose
+func (sm *DockerServiceManager) RunCommand(stack string, composeNames []string, composeArgs []string, env map[string]string) error {
+	return executeCompose(sm, true, composeNames, composeArgs, env)
+}
+
 // RunCompose runs a docker compose by its name
 func (sm *DockerServiceManager) RunCompose(isStack bool, composeNames []string, env map[string]string) error {
 	return executeCompose(sm, isStack, composeNames, []string{"up", "-d"}, env)
diff --git a/e2e/Makefile b/e2e/Makefile
@@ -32,14 +32,6 @@ functional-test:
 	OP_STACK_VERSION=${STACK_VERSION} \
 	go test -v --godog.format=${FORMAT} ${FEATURE}
 
-.PHONY: run-elastic-stack
-run-elastic-stack:
-	OP_LOG_LEVEL=${LOG_LEVEL} ./op run stack metricbeat -v ${STACK_VERSION}
-
-.PHONY: shutdown-elastic-stack
-shutdown-elastic-stack:
-	OP_LOG_LEVEL=${LOG_LEVEL} ./op stop stack metricbeat
-
 .PHONY: sync-integrations
 sync-integrations:
 	OP_LOG_LEVEL=${LOG_LEVEL} ./op sync integrations --delete
diff --git a/e2e/README.md b/e2e/README.md
@@ -103,9 +103,7 @@ $ # export GOOS=darwin                             # exports your O.S. (default
 $ # export GOARCH=amd64                            # exports your O.S. (default 'amd64', valid: [amd64, 386])
 $ make -C e2e install                              # installs tests dependencies
 $ make -C e2e fetch-binary                         # generates the binary from the repository
-$ make -C e2e run-elastic-stack                    # runs the stack for metricbeat
 $ make -C e2e functional-test                      # runs the test suite for Redis and stack 
-$ make -C e2e shutdown-elastic-stack               # stops the stack
 ```
 
 or simply run as the CI does:
@@ -146,3 +144,12 @@ go test -v --godog.format=${FORMAT} redis
 ```
 
 >For environment variables reference affecting the logs, please check out [CLI's docs](../cli/README.md#logging)
+
+## Debugging the tests
+
+### VSCode
+When using VSCode as editor, it's possible to debug the project using the existing VSCode configurations for debug.
+
+In order to debug the `godog` tests, 1) you must have the `runner_test.go` file opened as the current file in the IDE, 2) Use the Run/Debug module of VSCode, and 3) select the `Godog Tests` debug configuration to be executed.
+
+![](./debug.png)
diff --git a/e2e/assertions.go b/e2e/assertions.go
@@ -12,8 +12,8 @@ func assertHitsArePresent(hits map[string]interface{}, q ElasticsearchQuery) err
 	hitsCount := len(hits["hits"].(map[string]interface{})["hits"].([]interface{}))
 	if hitsCount == 0 {
 		return fmt.Errorf(
-			"There aren't documents for %s-%s on Metricbeat index",
-			q.EventModule, q.ServiceVersion)
+			"There aren't documents for %s-%s on Metricbeat index %s",
+			q.EventModule, q.ServiceVersion, q.IndexName)
 	}
 
 	return nil
@@ -33,8 +33,8 @@ func assertHitsDoNotContainErrors(hits map[string]interface{}, q ElasticsearchQu
 				}).Error("Error Hit found")
 
 				return fmt.Errorf(
-					"There are errors for %s-%s on Metricbeat index",
-					q.EventModule, q.ServiceVersion)
+					"There are errors for %s-%s on Metricbeat index %s",
+					q.EventModule, q.ServiceVersion, q.IndexName)
 			}
 		}
 	}
diff --git a/e2e/debug.png b/e2e/debug.png
diff --git a/e2e/elasticsearch.go b/e2e/elasticsearch.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"time"
 
+	backoff "github.com/cenkalti/backoff/v4"
 	es "github.com/elastic/go-elasticsearch/v8"
 	log "github.com/sirupsen/logrus"
 
@@ -18,6 +19,7 @@ import (
 //nolint:unused
 type ElasticsearchQuery struct {
 	EventModule    string
+	IndexName      string
 	ServiceVersion string
 }
 
@@ -44,7 +46,7 @@ func deleteIndex(ctx context.Context, stackName string, index string) error {
 	}
 	log.WithFields(log.Fields{
 		"indexName": index,
-		"status":    res.Status,
+		"status":    res.Status(),
 	}).Debug("Index deleted using Elasticsearch Go client")
 
 	res, err = esClient.Indices.DeleteAlias([]string{index}, []string{index})
@@ -58,7 +60,7 @@ func deleteIndex(ctx context.Context, stackName string, index string) error {
 	}
 	log.WithFields(log.Fields{
 		"indexAlias": index,
-		"status":     res.Status,
+		"status":     res.Status(),
 	}).Debug("Index Alias deleted using Elasticsearch Go client")
 
 	return nil
@@ -144,6 +146,7 @@ func search(stackName string, indexName string, query map[string]interface{}) (s
 	}
 
 	log.WithFields(log.Fields{
+		"index": indexName,
 		"query": fmt.Sprintf("%s", query),
 	}).Debug("Elasticsearch query")
 
@@ -156,7 +159,7 @@ func search(stackName string, indexName string, query map[string]interface{}) (s
 	if err != nil {
 		log.WithFields(log.Fields{
 			"error": err,
-		}).Error("Error getting response from Elasticsearch")
+		}).Error("Error performing search on Elasticsearch")
 
 		return result, err
 	}
@@ -172,17 +175,11 @@ func search(stackName string, indexName string, query map[string]interface{}) (s
 			return result, err
 		}
 
-		log.WithFields(log.Fields{
-			"status": res.Status(),
-			"type":   e["error"].(map[string]interface{})["type"],
-			"reason": e["error"].(map[string]interface{})["reason"],
-		}).Error("Error getting response from Elasticsearch")
-
-		return result, fmt.Errorf(
-			"Error getting response from Elasticsearch. Status: %s, Type: %s, Reason: %s",
-			res.Status(),
-			e["error"].(map[string]interface{})["type"],
-			e["error"].(map[string]interface{})["reason"])
+		err := fmt.Errorf(
+			"Error getting response from Elasticsearch. Status: %s, ResponseError: %v",
+			res.Status(), e)
+
+		return result, err
 	}
 
 	if err := json.NewDecoder(res.Body).Decode(&result); err != nil {
@@ -201,3 +198,49 @@ func search(stackName string, indexName string, query map[string]interface{}) (s
 
 	return result, nil
 }
+
+// waitForElasticsearchHealthy waits for elasticsearch to be healthy, returning false
+// if elasticsearch does not get healthy status in a defined number of minutes.
+//nolint:unused
+func waitForElasticsearch(maxTimeoutMinutes time.Duration, stackName string) (bool, error) {
+	exp := getExponentialBackOff(maxTimeoutMinutes)
+
+	retryCount := 1
+
+	clusterStatus := func() error {
+		esClient, err := getElasticsearchClient(stackName)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"error": err,
+			}).Warn("Could not obtain an Elasticsearch client")
+
+			return err
+		}
+
+		if _, err := esClient.Cluster.Health(); err != nil {
+			log.WithFields(log.Fields{
+				"error":       err,
+				"retry":       retryCount,
+				"elapsedTime": exp.GetElapsedTime(),
+			}).Warn("The Elasticsearch cluster is not healthy yet")
+
+			retryCount++
+
+			return err
+		}
+
+		log.WithFields(log.Fields{
+			"retries":     retryCount,
+			"elapsedTime": exp.GetElapsedTime(),
+		}).Info("The Elasticsearch cluster is healthy")
+
+		return nil
+	}
+
+	err := backoff.Retry(clusterStatus, exp)
+	if err != nil {
+		return false, err
+	}
+
+	return true, nil
+}
diff --git a/e2e/features/metricbeat/metricbeat.feature b/e2e/features/metricbeat/metricbeat.feature
@@ -2,11 +2,10 @@ Feature: As a Metricbeat developer I want to check that default configuration wo
 
 Scenario Outline: Check <configuration> configuration is sending metrics to Elasticsearch without errors
   Given metricbeat is installed using "<configuration>" configuration
-  When metricbeat runs for "20" seconds
+  When metricbeat runs for "30" seconds
   Then there are "system" events in the index
     And there are no errors in the index
 Examples:
 | configuration        |
 | metricbeat           |
 | metricbeat.docker    |
-| metricbeat.reference |
diff --git a/e2e/go.mod b/e2e/go.mod
@@ -3,13 +3,15 @@ module github.com/elastic/metricbeat-tests-poc/e2e
 go 1.12
 
 require (
+	github.com/cenkalti/backoff/v4 v4.0.2
 	github.com/cucumber/godog v0.9.0
 	github.com/cucumber/messages-go/v10 v10.0.3
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20190731061900-ea052088db25
 	github.com/elastic/metricbeat-tests-poc/cli v0.1.0-rc12
 	github.com/onsi/ginkgo v1.10.1 // indirect
 	github.com/onsi/gomega v1.7.0 // indirect
 	github.com/sirupsen/logrus v1.4.2
+	github.com/stretchr/testify v1.5.1
 )
 
 replace github.com/testcontainers/testcontainers-go v0.0.8 => github.com/mdelapenya/testcontainers-go v0.0.9-compose-1
diff --git a/e2e/go.sum b/e2e/go.sum
@@ -20,6 +20,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY
 github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
+github.com/cenkalti/backoff/v4 v4.0.2 h1:JIufpQLbh4DkbQoii76ItQIUFzevQSqOLZca4eamEDs=
+github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
diff --git a/e2e/metricbeat-poc.md b/e2e/metricbeat-poc.md
@@ -75,9 +75,9 @@ We are using Godog + Cucumber to implement the tests, where we create connection
 
 As an example, the Golang implementation of the `features/metricbeat/mysql.feature` is located under the [./metricbeat_test.go](./metricbeat_test.go) file.
 
-Each module will define its own file for specificacions, adding specific tags that will allow filtering the execution, if needed. These tags would be named after the module, so it will simplify the execution of just a module.
+Each module will define its own file for specificacions, adding specific feature context functions that will allow filtering the execution, if needed. These functions would be managed by a map in the test runner. We see this as a workaround to be improved in a future refactor, but we had to continue the development of the PoC while Godog solves the coupling of the life cycle hooks for each test suite, as demonstrated [here](https://github.com/mdelapenya/sample-godog). 
 
 ```shell
-# Will run all scenarios tagged with @mysql
-$ GO111MODULE=on godog -t mysql
+# Will run all scenarios for mysql
+$ GO111MODULE=on go test --godog.format pretty mysql
 ```
diff --git a/e2e/metricbeat_test.go b/e2e/metricbeat_test.go
diff --git a/e2e/utils.go b/e2e/utils.go

Original file line number	Diff line number	Diff line change
`@@ -102,12 +102,12 @@`
`102`	`102`	`"type": "go",`
`103`	`103`	`"request": "launch",`
`104`	`104`	`"mode": "auto",`
`105`		`- "program": "${workspaceFolder}/e2e",`
	`105`	`+ "program": "${file}",`
`106`	`106`	`"env": {`
`107`	`107`	`"GO111MODULE": "on",`
`108`	`108`	`"OP_LOG_LEVEL": "DEBUG",`
`109`	`109`	`},`
`110`		`- "args": ["godog", "-t", "mysql"]`
	`110`	`+ "args": ["--godog.format", "pretty", "metricbeat"]`
`111`	`111`	`}`
`112`	`112`	`]`
`113`	`113`	`}`