Merge pull request #104 from mdelapenya/use-go-test-as-runner

mdelapenya · web-flow · commit 9348b71ecfad · 2020-04-16T20:04:59.000+02:00
Use "go test" as test runner
diff --git a/.ci/scripts/functional-test.sh b/.ci/scripts/functional-test.sh
@@ -35,10 +35,8 @@ rm -rf outputs || true
 mkdir -p outputs
 
 ## Parse FEATURE if not ALL then enable the flags to be passed to the functional-test wrapper
-FLAG=''
 REPORT=''
 if [ "${FEATURE}" != "" ] && [ "${FEATURE}" != "all" ] ; then
-  FLAG='-t'
   REPORT=outputs/TEST-${FEATURE}
 else
   FEATURE=''
@@ -48,11 +46,12 @@ fi
 ## Generate test report even if make failed.
 set +e
 exit_status=0
-if ! FLAG=${FLAG} FEATURE=${FEATURE} FORMAT=junit STACK_VERSION=${STACK_VERSION} METRICBEAT_VERSION=${METRICBEAT_VERSION} make --no-print-directory -C e2e functional-test | tee ${REPORT}  ; then
+if ! FEATURE=${FEATURE} FORMAT=junit STACK_VERSION=${STACK_VERSION} METRICBEAT_VERSION=${METRICBEAT_VERSION} make --no-print-directory -C e2e functional-test | tee ${REPORT}  ; then
   echo 'ERROR: functional-test failed'
   exit_status=1
 fi
 
 ## Transform report to Junit by parsing the stdout generated previously
+sed -i "s/testing: warning: no tests to run//" ${REPORT}
 sed -e 's/^[ \t]*//; s#>.*failed$#>#g' ${REPORT} | grep -E '^<.*>$' > ${REPORT}.xml
 exit $exit_status
diff --git a/e2e/Makefile b/e2e/Makefile
@@ -8,10 +8,6 @@ STACK_VERSION?=
 METRICBEAT_VERSION?=
 VERSION_VALUE=`cat ../cli/VERSION.txt`
 
-ifneq ($(FEATURE),)
-FEATURE_FLAG=--tags
-endif
-
 GO_IMAGE_TAG?='stretch'
 GOOS?='linux'
 GOARCH?='amd64'
@@ -26,20 +22,15 @@ fetch-binary:
 install:
 	go get -v -t ./...
 
-.PHONY: install-godog
-install-godog: export GO111MODULE := on
-install-godog:
-	go get -v github.com/cucumber/godog/cmd/godog@v0.9.0
-
 .PHONY: functional-test
-functional-test: install-godog
+functional-test:
 	OP_LOG_LEVEL=${LOG_LEVEL} \
 	OP_LOG_INCLUDE_TIMESTAMP=${LOG_INCLUDE_TIMESTAMP} \
 	OP_QUERY_MAX_ATTEMPTS=${QUERY_MAX_ATTEMPTS} \
 	OP_RETRY_TIMEOUT=${RETRY_TIMEOUT} \
 	OP_METRICBEAT_VERSION=${METRICBEAT_VERSION} \
 	OP_STACK_VERSION=${STACK_VERSION} \
-	godog --format=${FORMAT} ${FEATURE_FLAG} ${FEATURE}
+	go test -v --godog.format=${FORMAT} ${FEATURE}
 
 .PHONY: run-elastic-stack
 run-elastic-stack:
diff --git a/e2e/README.md b/e2e/README.md
@@ -52,7 +52,7 @@ Ok, you want to contribute the tests for a new integration module. Then you have
 - A `configuration file`, in YAML format, with any Metricbeat configuration that is specific to the module.
 
 ### Feature files
-We will create use cases for the module in a separate `.feature` file, ideally named after module's name (i.e. _apache.feature_). This feature file is a Cucumber requirement, that will be parsed by the test runner and matched against the Golang code implementing the tests.
+We will create use cases for the module in a separate `.feature` file, ideally named after module's name (i.e. _apache.feature_), and located under the `metricbeat` features directory. This feature file is a Cucumber requirement, that will be parsed by the test runner and matched against the Golang code implementing the tests.
 
 ```cucumber
 @apache
@@ -61,15 +61,17 @@ Feature: As a Metricbeat developer I want to check that the Apache module works
 Scenario Outline: Check module is sending metrics to Elasticsearch without errors
   Given Apache "<apache_version>" is running for metricbeat
     And metricbeat is installed and configured for Apache module
+    And metricbeat waits "20" seconds for the service
+  When metricbeat runs for "20" seconds 
   Then there are "Apache" events in the index
     And there are no errors in the index
 Examples:
 | apache_version |
-| 2.2            |
-| 2.4            |
+| 2.4.12         |
+| 2.4.20         |
 ```
 
->You should write as many scenarios as you considering, covering different use cases in each scenario, taking care of duplicated steps that could be reused by other module.
+>You should write as many scenarios as you consider, covering different use cases in each scenario, taking care of duplicated steps that could be reused by other module.
 
 The anatomy of a feature file is:
 
@@ -110,8 +112,8 @@ or simply run as the CI does:
 
 ```shell
 $ export GO_VERSION=1.12.7                         # exports which GIMME version to use
-$ export STACK_VERSION=7.5.0                       # exports stack version as runtime
-$ export METRICBEAT_VERSION=7.5.0                  # exports metricbeat version
+$ export STACK_VERSION=7.6.0                       # exports stack version as runtime
+$ export METRICBEAT_VERSION=7.6.0                  # exports metricbeat version
 $ #export FEATURE=redis                            # exports which feature to run (default 'all')
 $ ./.ci/scripts/functional-test.sh ${GO_VERSION} ${FEATURE}
 ```
@@ -140,7 +142,7 @@ export OP_QUERY_MAX_ATTEMPTS=${OP_QUERY_MAX_ATTEMPTS:-5}
 export OP_RETRY_TIMEOUT=${OP_RETRY_TIMEOUT:-3}
 export FORMAT=${FORMAT:-pretty} # valid formats are: pretty, junit
 # If you do not pass a '-t moduleName' argument, then all tests will be run
-godog --format=${FORMAT} -t redis
+go test -v --godog.format=${FORMAT} redis
 ```
 
 >For environment variables reference affecting the logs, please check out [CLI's docs](../cli/README.md#logging)
diff --git a/e2e/features/apm/helm.feature b/e2e/features/apm/helm.feature
@@ -1,6 +1,3 @@
-@helm
-@k8s
-@apm
 Feature: The Helm chart is following product recommended configuration for Kubernetes
 
 Scenario: The APM Server chart will create recommended K8S resources
diff --git a/e2e/features/filebeat/helm.feature b/e2e/features/filebeat/helm.feature
@@ -1,6 +1,3 @@
-@helm
-@k8s
-@filebeat
 Feature: The Helm chart is following product recommended configuration for Kubernetes
 
 Scenario: The Filebeat chart will create recommended K8S resources
diff --git a/e2e/features/metricbeat/apache.feature b/e2e/features/metricbeat/apache.feature
@@ -1,4 +1,3 @@
-@apache
 Feature: As a Metricbeat developer I want to check that the Apache module works as expected
 
 Scenario Outline: Check module is sending metrics to Elasticsearch without errors
diff --git a/e2e/features/metricbeat/helm.feature b/e2e/features/metricbeat/helm.feature
@@ -1,6 +1,3 @@
-@helm
-@k8s
-@metricbeat
 Feature: The Helm chart is following product recommended configuration for Kubernetes
 
 Scenario: The Metricbeat chart will create recommended K8S resources
diff --git a/e2e/features/metricbeat/metricbeat.feature b/e2e/features/metricbeat/metricbeat.feature
@@ -1,4 +1,3 @@
-@metricbeat
 Feature: As a Metricbeat developer I want to check that default configuration works as expected
 
 Scenario Outline: Check <configuration> configuration is sending metrics to Elasticsearch without errors
diff --git a/e2e/features/metricbeat/mysql.feature b/e2e/features/metricbeat/mysql.feature
@@ -1,4 +1,3 @@
-@mysql
 Feature: As a Metricbeat developer I want to check that the MySQL module works as expected
 
 Scenario Outline: Check module is sending metrics to Elasticsearch without errors
diff --git a/e2e/features/metricbeat/vsphere.feature b/e2e/features/metricbeat/vsphere.feature
@@ -1,4 +1,3 @@
-@vsphere
 Feature: As a Metricbeat developer I want to check that the vSphere module works as expected
 
 Scenario Outline: Check module is sending metrics to Elasticsearch without errors
diff --git a/e2e/metricbeat-poc.md b/e2e/metricbeat-poc.md
@@ -8,10 +8,10 @@ So for that reason we are adding [smoke tests](http://softwaretestingfundamental
 
 ## Running the tests
 
-The tests are located under the [./](root) directory. Place your terminal there and execute `godog`, which is the official Golang implementation for Cucumber:
+The tests are located under the [root](./) directory. Place your terminal there and execute the Godog tests, which is the official Golang implementation for Cucumber:
 
 ```shell
-$ GO111MODULE=on godog
+$ GO111MODULE=on go test -v --godog.format pretty [apache|filebeat|helm|metricbeat|mysql|redis|vsphere]
 ```
 
 ## Tooling
@@ -46,28 +46,34 @@ For this POC, we have chosen Godog over any other test framework because the tea
 
 All the Gherkin (Cucumber) specifications are written in `.feature` files.
 
-A good example could be [this one](./features/mysql.feature):
+A good example could be [this one](./features/metricbeat/mysql.feature):
 
 ```cucumber
-@mysql
 Feature: As a Metricbeat developer I want to check that the MySQL module works as expected
 
-Scenario Outline: Check module is sending metrics to a file
-  Given MySQL "<mysql_version>" is running on port "3306"
-    And metricbeat "7.2.0" is installed and configured for MySQL module
-  Then metricbeat outputs metrics to the file "mysql-<mysql_version>.metrics"
+Scenario Outline: Check module is sending metrics to Elasticsearch without errors
+  Given "<variant>" v<version>, variant of "MySQL", is running for metricbeat
+    And metricbeat is installed and configured for "<variant>", variant of the "MySQL" module
+    And metricbeat waits "20" seconds for the service
+  When metricbeat runs for "20" seconds
+  Then there are "<variant>" events in the index
+    And there are no errors in the index
 Examples:
-| mysql_version |
-| 5.6  |
-| 5.7  |
-| 8.0  |
+| variant | version    |
+| MariaDB | 10.2.23    |
+| MariaDB | 10.3.14    |
+| MariaDB | 10.4.4     |
+| MySQL   | 5.7.12     |
+| MySQL   | 8.0.13     |
+| Percona | 5.7.24     |
+| Percona | 8.0.13-4   |
 ```
 
 ## Test Implementation
 
 We are using Godog + Cucumber to implement the tests, where we create connections to the `Given`, `When`, `Then`, `And`, etc. in a well-known file structure.
 
-As an example, the Golang implementation of the `features/mysql.feature` is located under the [./mysql_test.go](./mysql_test.go) file.
+As an example, the Golang implementation of the `features/metricbeat/mysql.feature` is located under the [./metricbeat_test.go](./metricbeat_test.go) file.
 
 Each module will define its own file for specificacions, adding specific tags that will allow filtering the execution, if needed. These tags would be named after the module, so it will simplify the execution of just a module.
 
diff --git a/e2e/runner_test.go b/e2e/runner_test.go
@@ -3,15 +3,77 @@ package e2e
 import (
 	"flag"
 	"os"
+	"path"
 	"strconv"
 	"testing"
 
 	"github.com/cucumber/godog"
 	"github.com/cucumber/godog/colors"
+	log "github.com/sirupsen/logrus"
 
 	"github.com/elastic/metricbeat-tests-poc/cli/config"
 )
 
+type contextMetadata struct {
+	name         string
+	modules      []string
+	contextFuncs []func(s *godog.Suite) // the functions that hold the steps for a specific
+}
+
+func (c *contextMetadata) getFeaturePaths() []string {
+	paths := []string{}
+	for _, module := range c.modules {
+		paths = append(paths, path.Join("features", module, c.name+".feature"))
+	}
+
+	return paths
+}
+
+var supportedProducts = map[string]*contextMetadata{
+	"apache": &contextMetadata{
+		contextFuncs: []func(s *godog.Suite){
+			MetricbeatFeatureContext,
+		},
+		modules: []string{"metricbeat"},
+	},
+	"helm": &contextMetadata{
+		contextFuncs: []func(s *godog.Suite){
+			HelmChartFeatureContext,
+		},
+		modules: []string{"apm", "filebeat", "metricbeat"},
+	},
+	"filebeat": &contextMetadata{
+		contextFuncs: []func(s *godog.Suite){
+			HelmChartFeatureContext,
+		},
+		modules: []string{"filebeat"},
+	},
+	"metricbeat": &contextMetadata{
+		contextFuncs: []func(s *godog.Suite){
+			MetricbeatFeatureContext,
+		},
+		modules: []string{"metricbeat"},
+	},
+	"mysql": &contextMetadata{
+		contextFuncs: []func(s *godog.Suite){
+			MetricbeatFeatureContext,
+		},
+		modules: []string{"metricbeat"},
+	},
+	"redis": &contextMetadata{
+		contextFuncs: []func(s *godog.Suite){
+			MetricbeatFeatureContext,
+		},
+		modules: []string{"metricbeat"},
+	},
+	"vsphere": &contextMetadata{
+		contextFuncs: []func(s *godog.Suite){
+			MetricbeatFeatureContext,
+		},
+		modules: []string{"metricbeat"},
+	},
+}
+
 // stackVersion is the version of the stack to use
 // It can be overriden by OP_STACK_VERSION env var
 var stackVersion = "7.6.0"
@@ -58,12 +120,67 @@ func init() {
 
 func TestMain(m *testing.M) {
 	flag.Parse()
-	opt.Paths = []string{"features/*"}
 
-	status := godog.RunWithOptions("godog", func(s *godog.Suite) {}, opt)
+	featurePaths, metadatas := parseFeatureFlags(flag.Args())
+
+	if len(metadatas) == 0 {
+		log.Error("We did not find anything to execute. Exiting")
+		os.Exit(1)
+	}
+
+	opt.Paths = featurePaths
+
+	status := godog.RunWithOptions("godog", func(s *godog.Suite) {
+		for _, metadata := range metadatas {
+			for _, f := range metadata.contextFuncs {
+				f(s)
+			}
+		}
+	}, opt)
 
 	if st := m.Run(); st > status {
 		status = st
 	}
 	os.Exit(status)
 }
+
+func findSupportedContext(feature string) *contextMetadata {
+	for k, ctx := range supportedProducts {
+		ctx.name = k // match key with context name
+		if k == feature {
+			log.WithFields(log.Fields{
+				"paths":   ctx.getFeaturePaths(),
+				"modules": ctx.modules,
+			}).Info("Feature Context found")
+			return ctx
+		}
+	}
+
+	return nil
+}
+
+func parseFeatureFlags(flags []string) ([]string, []*contextMetadata) {
+	metadatas := []*contextMetadata{}
+	featurePaths := []string{}
+
+	if len(flags) == 1 && flags[0] == "all" {
+		for k, metadata := range supportedProducts {
+			metadata.name = k // match key with context name
+			metadatas = append(metadatas, metadata)
+		}
+	} else {
+		for _, feature := range flags {
+			metadata := findSupportedContext(feature)
+
+			if metadata == nil {
+				log.Warnf("Sorry but we don't support tests for %s at this moment. Skipping it :(", feature)
+				continue
+			}
+
+			metadatas = append(metadatas, metadata)
+			featurePaths = append(featurePaths, metadata.getFeaturePaths()...)
+		}
+	}
+
+	return featurePaths, metadatas
+}

-Original file line number
+Diff line change
@@ @@ -1,6 +1,3 @@ @@
 -@helm
 -@k8s
 -@apm
 Feature: The Helm chart is following product recommended configuration for Kubernetes
 Scenario: The APM Server chart will create recommended K8S resources
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-@apache`
`2`	`1`	`Feature: As a Metricbeat developer I want to check that the Apache module works as expected`
`3`	`2`
`4`	`3`	`Scenario Outline: Check module is sending metrics to Elasticsearch without errors`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-@metricbeat`
`2`	`1`	`Feature: As a Metricbeat developer I want to check that default configuration works as expected`
`3`	`2`
`4`	`3`	`Scenario Outline: Check <configuration> configuration is sending metrics to Elasticsearch without errors`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-@mysql`
`2`	`1`	`Feature: As a Metricbeat developer I want to check that the MySQL module works as expected`
`3`	`2`
`4`	`3`	`Scenario Outline: Check module is sending metrics to Elasticsearch without errors`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-@vsphere`
`2`	`1`	`Feature: As a Metricbeat developer I want to check that the vSphere module works as expected`
`3`	`2`
`4`	`3`	`Scenario Outline: Check module is sending metrics to Elasticsearch without errors`