From 944869e68b42d314a1cabcf3860ca94126f2260c Mon Sep 17 00:00:00 2001 From: "an.makarov" Date: Sun, 16 Nov 2025 09:32:24 +0300 Subject: [PATCH] parent 843c768e333c551c7534f0370864a30b80400cf1 author an.makarov 1763274744 +0300 committer an.makarov 1763904905 +0300 `configcheck` validates ClusterVectorPipeline (CVP) against all ClusterVectorAggregator instances instead of only matching ones Fixes #201 feat: add e2e test framework and CI workflow (#206) * feat: add e2e test framework and CI workflow - Add comprehensive e2e test framework with Ginkgo/Gomega - Add GitHub Actions workflow for automated e2e testing - Add artifact collection for test failures - Add HTML report generation script - Add normal mode tests as baseline - Update Makefile with test-e2e, test-report, deploy-helm-e2e targets - Update Ginkgo to v2.20.2 and Gomega to v1.34.1 - Add CI/CD documentation * fix(makefile): add ginkgo dependency for test-e2e target - Add GINKGO binary to tool binaries section - Add GINKGO_VERSION (v2.20.2) to tool versions - Add ginkgo target to download ginkgo if necessary - Update test-e2e target to depend on ginkgo and use $(GINKGO) This fixes CI failure where ginkgo command was not found. * chore(ci): remove event-collector from e2e workflow Event collector is not used in current e2e tests. * fix(makefile): use absolute path for ginkgo binary Formatted k8s package Fixed imports for linting :) Fix imports again :) --- .github/workflows/e2e-tests.yaml | 76 + .github/workflows/lint.yaml | 67 + .gitignore | 3 + .golangci.yml | 25 + Makefile | 86 +- api/v1alpha1/clustervectorpipeline.go | 4 +- api/v1alpha1/vectorpipeline.go | 4 +- api/v1alpha1/zz_generated.deepcopy.go | 2 +- cmd/event_collector/main.go | 18 +- cmd/evgen/main.go | 7 +- cmd/manager/main.go | 3 +- docs/ci-cd.md | 174 + docs/specification.md | 2 +- go.mod | 20 +- go.sum | 40 +- helm/charts/vector-operator/Chart.yaml | 4 +- helm/index.yaml | 107 +- helm/packages/vector-operator-0.7.2.tgz | Bin 0 -> 102637 bytes internal/config/agent.go | 8 +- internal/config/aggregator.go | 10 +- internal/config/config.go | 12 +- internal/config/configcheck/configcheck.go | 7 +- .../config/configcheck/configcheck_config.go | 3 +- .../config/configcheck/configcheck_pod.go | 54 +- internal/config/default.go | 11 +- internal/config/types.go | 1 + .../clustervectoraggregator_controller.go | 16 +- ...clustervectoraggregator_controller_test.go | 1 + internal/controller/pipeline_controller.go | 35 +- .../controller/pipeline_controller_test.go | 1 + internal/controller/suite_test.go | 4 +- internal/controller/vector_controller.go | 6 +- internal/controller/vector_controller_test.go | 1 + .../controller/vectoraggregator_controller.go | 16 +- .../vectoraggregator_controller_test.go | 3 +- internal/evcollector/collector.go | 6 +- internal/evcollector/event.go | 6 +- internal/pipeline/hash.go | 1 + internal/pipeline/pipeline.go | 20 +- internal/pipeline/pipeline_test.go | 59 - internal/utils/hash/hash_test.go | 3 +- internal/utils/k8s/k8s_test.go | 3 +- internal/utils/k8s/label.go | 41 +- internal/utils/k8s/label_test.go | 114 + internal/vector/aggregator/config.go | 6 +- internal/vector/aggregator/controller.go | 9 +- internal/vector/aggregator/deployment.go | 65 +- internal/vector/aggregator/event_collector.go | 5 +- internal/vector/aggregator/podmonitor.go | 3 +- internal/vector/aggregator/rbac.go | 4 +- internal/vector/aggregator/service.go | 3 +- internal/vector/gen/event.pb.go | 5 +- internal/vector/gen/vector.pb.go | 5 +- internal/vector/gen/vector_grpc.pb.go | 1 + internal/vector/vectoragent/vectoragent.go | 6 +- .../vector/vectoragent/vectoragent_config.go | 3 +- .../vectoragent/vectoragent_controller.go | 6 +- .../vectoragent/vectoragent_daemonset.go | 61 +- .../vector/vectoragent/vectoragent_default.go | 3 +- .../vector/vectoragent/vectoragent_service.go | 3 +- scripts/kind-config-ci.yaml | 18 + test/e2e/e2e_suite_test.go | 345 ++ test/e2e/e2e_test.go | 58 +- test/e2e/framework/README.md | 612 +++ test/e2e/framework/artifacts/README.md | 178 + test/e2e/framework/artifacts/collector.go | 621 +++ test/e2e/framework/artifacts/config.go | 137 + test/e2e/framework/artifacts/metadata.go | 118 + test/e2e/framework/artifacts/storage.go | 327 ++ test/e2e/framework/artifacts/storage_test.go | 55 + test/e2e/framework/assertions/matchers.go | 300 ++ test/e2e/framework/config/constants.go | 50 + test/e2e/framework/config/timeouts.go | 92 + test/e2e/framework/errors/errors.go | 100 + test/e2e/framework/framework.go | 1154 +++++ test/e2e/framework/kubectl/client.go | 485 ++ test/e2e/framework/kubectl/validation.go | 143 + test/e2e/framework/kubectl/wait.go | 111 + test/e2e/framework/lifecycle.go | 91 + test/e2e/framework/recorder/recorder.go | 258 + test/e2e/framework/resources.go | 36 + test/e2e/normal_mode_e2e_test.go | 222 + test/e2e/scripts/README.md | 40 + test/e2e/scripts/generate_report.py | 4554 +++++++++++++++++ test/e2e/testdata/normal-mode/agent.yaml | 7 + test/e2e/testdata/normal-mode/aggregator.yaml | 11 + .../normal-mode/cluster-pipeline-pod-ns1.yaml | 20 + .../normal-mode/cluster-pipeline-pod-ns2.yaml | 21 + .../normal-mode/cluster-pipeline.yaml | 24 + .../normal-mode/namespace-isolation-ns.yaml | 4 + .../namespace-isolation-pipeline.yaml | 17 + .../namespace-isolation-pod-isolated.yaml | 20 + .../namespace-isolation-pod-main.yaml | 19 + .../normal-mode/pipeline-aggregator-role.yaml | 28 + .../testdata/normal-mode/pipeline-basic.yaml | 16 + .../normal-mode/pipeline-complex.yaml | 34 + .../normal-mode/pipeline-deletable.yaml | 16 + .../normal-mode/pipeline-kubernetes-logs.yaml | 24 + .../normal-mode/pipeline-template.yaml | 16 + .../testdata/normal-mode/test-app-pod.yaml | 20 + test/e2e/testdata_helper.go | 17 + 101 files changed, 11339 insertions(+), 352 deletions(-) create mode 100644 .github/workflows/e2e-tests.yaml create mode 100644 .github/workflows/lint.yaml create mode 100644 .golangci.yml create mode 100644 docs/ci-cd.md create mode 100644 helm/packages/vector-operator-0.7.2.tgz delete mode 100644 internal/pipeline/pipeline_test.go create mode 100644 internal/utils/k8s/label_test.go create mode 100644 scripts/kind-config-ci.yaml create mode 100644 test/e2e/framework/README.md create mode 100644 test/e2e/framework/artifacts/README.md create mode 100644 test/e2e/framework/artifacts/collector.go create mode 100644 test/e2e/framework/artifacts/config.go create mode 100644 test/e2e/framework/artifacts/metadata.go create mode 100644 test/e2e/framework/artifacts/storage.go create mode 100644 test/e2e/framework/artifacts/storage_test.go create mode 100644 test/e2e/framework/assertions/matchers.go create mode 100644 test/e2e/framework/config/constants.go create mode 100644 test/e2e/framework/config/timeouts.go create mode 100644 test/e2e/framework/errors/errors.go create mode 100644 test/e2e/framework/framework.go create mode 100644 test/e2e/framework/kubectl/client.go create mode 100644 test/e2e/framework/kubectl/validation.go create mode 100644 test/e2e/framework/kubectl/wait.go create mode 100644 test/e2e/framework/lifecycle.go create mode 100644 test/e2e/framework/recorder/recorder.go create mode 100644 test/e2e/framework/resources.go create mode 100644 test/e2e/normal_mode_e2e_test.go create mode 100644 test/e2e/scripts/README.md create mode 100644 test/e2e/scripts/generate_report.py create mode 100644 test/e2e/testdata/normal-mode/agent.yaml create mode 100644 test/e2e/testdata/normal-mode/aggregator.yaml create mode 100644 test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns1.yaml create mode 100644 test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns2.yaml create mode 100644 test/e2e/testdata/normal-mode/cluster-pipeline.yaml create mode 100644 test/e2e/testdata/normal-mode/namespace-isolation-ns.yaml create mode 100644 test/e2e/testdata/normal-mode/namespace-isolation-pipeline.yaml create mode 100644 test/e2e/testdata/normal-mode/namespace-isolation-pod-isolated.yaml create mode 100644 test/e2e/testdata/normal-mode/namespace-isolation-pod-main.yaml create mode 100644 test/e2e/testdata/normal-mode/pipeline-aggregator-role.yaml create mode 100644 test/e2e/testdata/normal-mode/pipeline-basic.yaml create mode 100644 test/e2e/testdata/normal-mode/pipeline-complex.yaml create mode 100644 test/e2e/testdata/normal-mode/pipeline-deletable.yaml create mode 100644 test/e2e/testdata/normal-mode/pipeline-kubernetes-logs.yaml create mode 100644 test/e2e/testdata/normal-mode/pipeline-template.yaml create mode 100644 test/e2e/testdata/normal-mode/test-app-pod.yaml create mode 100644 test/e2e/testdata_helper.go diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml new file mode 100644 index 00000000..cfc09b59 --- /dev/null +++ b/.github/workflows/e2e-tests.yaml @@ -0,0 +1,76 @@ +name: E2E Tests + +on: + push: + branches: + - main + - master + pull_request: + branches: + - main + - master + workflow_dispatch: # Allow manual trigger + +jobs: + e2e-tests: + name: Run E2E Tests + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + cache: true + + - name: Install dependencies + run: | + # Install kubebuilder for CRD generation + curl -L -o kubebuilder https://go.kubebuilder.io/dl/latest/$(go env GOOS)/$(go env GOARCH) + chmod +x kubebuilder && sudo mv kubebuilder /usr/local/bin/ + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: kind + config: scripts/kind-config-ci.yaml + wait: 300s + + - name: Verify cluster + run: | + kubectl cluster-info + kubectl get nodes + kubectl get pods -A + + - name: Build operator image + run: | + docker build -t example.com/vector-operator:v0.0.1 . + + - name: Load image into kind + run: | + kind load docker-image example.com/vector-operator:v0.0.1 --name kind + + - name: Run E2E tests + run: make test-e2e + env: + KUBECONFIG: /home/runner/.kube/config + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: e2e-results-${{ github.run_number }} + path: test/e2e/results/ + retention-days: 7 + + - name: Publish test results + if: always() + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + files: test/e2e/results/run-*/reports/junit-report.xml + check_name: E2E Test Results + comment_mode: off diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 00000000..b3841489 --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,67 @@ +name: Lint + +on: + push: + branches: [main] + pull_request: + branches: [main] + +permissions: + contents: read + +jobs: + golangci-lint: + name: golangci-lint + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v6 + with: + version: v1.64 + args: --timeout=5m + + go-fmt: + name: go fmt + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Check formatting + run: | + if [ -n "$(gofmt -l .)" ]; then + echo "The following files are not formatted:" + gofmt -l . + exit 1 + fi + + go-vet: + name: go vet + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Run go vet + run: go vet ./... diff --git a/.gitignore b/.gitignore index 37afa32b..3e3a6cd5 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,6 @@ testbin/* __debug_bin vendor + +# E2E test results and artifacts +test/e2e/results/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..11c66482 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,25 @@ +run: + timeout: 5m + modules-download-mode: readonly + +linters: + enable: + - gosimple + - govet + - ineffassign + - staticcheck + - unused + - errcheck + - gofmt + - goimports + +linters-settings: + goimports: + local-prefixes: github.com/kaasops/vector-operator + +issues: + exclude-rules: + # Exclude some linters from running on tests files + - path: _test\.go + linters: + - errcheck diff --git a/Makefile b/Makefile index c05a018a..833ccf93 100644 --- a/Makefile +++ b/Makefile @@ -64,10 +64,79 @@ vet: ## Run go vet against code. test: manifests generate fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v +# E2E test configuration +E2E_FAIL_FAST ?= false +E2E_RUN_DESCRIPTION ?= +E2E_LABEL_FILTER ?= +NAMESPACE ?= vector + +.PHONY: test-e2e # Run e2e tests with comprehensive reporting (JUnit XML + JSON + logs + artifacts) +test-e2e: ginkgo + @TIMESTAMP=$$(date +%Y-%m-%d-%H%M%S); \ + RUN_DIR="test/e2e/results/run-$$TIMESTAMP"; \ + echo "==> Running e2e tests..."; \ + echo "==> Results will be saved to: $$RUN_DIR"; \ + mkdir -p "$$RUN_DIR/reports"; \ + export E2E_ARTIFACTS_DIR="$$RUN_DIR/artifacts"; \ + export E2E_ARTIFACTS_ENABLED=true; \ + export E2E_GIT_COMMIT=$$(git rev-parse HEAD 2>/dev/null || echo "unknown"); \ + export E2E_GIT_BRANCH=$$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown"); \ + export E2E_GIT_DIRTY=$$(git diff --quiet 2>/dev/null || echo "dirty"; git diff --cached --quiet 2>/dev/null || echo "staged"); \ + export E2E_RUN_DESCRIPTION="$(E2E_RUN_DESCRIPTION)"; \ + echo "==> Git info: commit=$$E2E_GIT_COMMIT branch=$$E2E_GIT_BRANCH dirty=$$E2E_GIT_DIRTY"; \ + if [ -n "$$E2E_RUN_DESCRIPTION" ]; then \ + echo "==> Run description: $$E2E_RUN_DESCRIPTION"; \ + fi; \ + GINKGO_FLAGS="-v -timeout=30m"; \ + if [ "$(E2E_FAIL_FAST)" = "true" ]; then \ + echo "==> Fail-fast mode enabled (stop on first failure)"; \ + GINKGO_FLAGS="$$GINKGO_FLAGS --fail-fast"; \ + fi; \ + if [ -n "$(E2E_LABEL_FILTER)" ]; then \ + echo "==> Label filter: $(E2E_LABEL_FILTER)"; \ + GINKGO_FLAGS="$$GINKGO_FLAGS --label-filter=\"$(E2E_LABEL_FILTER)\""; \ + fi; \ + cd test/e2e && $(GINKGO) $$GINKGO_FLAGS \ + --junit-report="../../$$RUN_DIR/reports/junit-report.xml" \ + --json-report="../../$$RUN_DIR/reports/report.json" \ + | tee "../../$$RUN_DIR/reports/test-output.log"; \ + EXIT_CODE=$$?; \ + echo ""; \ + echo "==> Test run complete!"; \ + echo "==> All results in one place: $$RUN_DIR"; \ + echo " Reports:"; \ + echo " - JUnit XML: $$RUN_DIR/reports/junit-report.xml"; \ + echo " - JSON: $$RUN_DIR/reports/report.json"; \ + echo " - Logs: $$RUN_DIR/reports/test-output.log"; \ + if [ -d "$$RUN_DIR/artifacts" ] && [ "$$(find $$RUN_DIR/artifacts -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" -gt 1 ]; then \ + echo " Artifacts: $$RUN_DIR/artifacts/ (collected for failed tests)"; \ + else \ + echo " Artifacts: None (all tests passed)"; \ + fi; \ + echo ""; \ + echo "Quick commands:"; \ + echo " View summary: cat $$RUN_DIR/artifacts/metadata.json 2>/dev/null || echo 'All tests passed'"; \ + echo " View failures: grep -A 5 'FAILED' $$RUN_DIR/reports/test-output.log 2>/dev/null || echo 'No failures'"; \ + exit $$EXIT_CODE + +.PHONY: test-report +test-report: ## Generate interactive HTML report from e2e test results + @echo "==> Generating test report..." + @cd test/e2e/results && python3 ../scripts/generate_report.py + @echo "==> Report generated: test/e2e/results/test_results_report.html" + +.PHONY: deploy-helm-e2e +deploy-helm-e2e: manifests ## Deploy operator using Helm for e2e tests (use IMG and NAMESPACE variables) + @echo "==> Installing CRDs..." + $(KUBECTL) apply -f config/crd/bases + @echo "==> Creating namespace $(NAMESPACE)..." + $(KUBECTL) create namespace $(NAMESPACE) || true + @echo "==> Deploying operator via Helm to namespace $(NAMESPACE)..." + helm upgrade --install vector-operator ./helm/charts/vector-operator \ + --namespace $(NAMESPACE) \ + --set image.repository=$$(echo $(IMG) | cut -d: -f1) \ + --set image.tag=$$(echo $(IMG) | cut -d: -f2) \ + --wait --timeout 5m .PHONY: lint lint: golangci-lint ## Run golangci-lint linter @@ -160,12 +229,14 @@ KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint +GINKGO ?= $(LOCALBIN)/ginkgo ## Tool Versions KUSTOMIZE_VERSION ?= v5.4.3 CONTROLLER_TOOLS_VERSION ?= v0.16.1 ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.59.1 +GOLANGCI_LINT_VERSION ?= v1.64.8 +GINKGO_VERSION ?= v2.20.2 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. @@ -187,6 +258,11 @@ golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) +.PHONY: ginkgo +ginkgo: $(GINKGO) ## Download ginkgo locally if necessary. +$(GINKGO): $(LOCALBIN) + $(call go-install-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo,$(GINKGO_VERSION)) + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed diff --git a/api/v1alpha1/clustervectorpipeline.go b/api/v1alpha1/clustervectorpipeline.go index da07cbe8..02a40016 100644 --- a/api/v1alpha1/clustervectorpipeline.go +++ b/api/v1alpha1/clustervectorpipeline.go @@ -2,10 +2,12 @@ package v1alpha1 import ( "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/kaasops/vector-operator/internal/utils/k8s" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kaasops/vector-operator/internal/utils/k8s" ) func (vp *ClusterVectorPipeline) GetSpec() VectorPipelineSpec { diff --git a/api/v1alpha1/vectorpipeline.go b/api/v1alpha1/vectorpipeline.go index 5fa963c0..a21b295c 100644 --- a/api/v1alpha1/vectorpipeline.go +++ b/api/v1alpha1/vectorpipeline.go @@ -2,10 +2,12 @@ package v1alpha1 import ( "context" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/kaasops/vector-operator/internal/utils/k8s" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kaasops/vector-operator/internal/utils/k8s" ) type VectorPipelineRole string diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 88f3346a..5b439271 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,7 +21,7 @@ limitations under the License. package v1alpha1 import ( - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) diff --git a/cmd/event_collector/main.go b/cmd/event_collector/main.go index ae2f07f0..8d986c0c 100644 --- a/cmd/event_collector/main.go +++ b/cmd/event_collector/main.go @@ -5,20 +5,22 @@ import ( "errors" "flag" "fmt" - "github.com/fsnotify/fsnotify" - "github.com/kaasops/vector-operator/internal/buildinfo" - "github.com/kaasops/vector-operator/internal/evcollector" - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/spf13/viper" - "k8s.io/client-go/kubernetes" "log/slog" "net" "net/http" "os" "os/signal" - ctrl "sigs.k8s.io/controller-runtime" "strings" "syscall" + + "github.com/fsnotify/fsnotify" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/spf13/viper" + "k8s.io/client-go/kubernetes" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/kaasops/vector-operator/internal/buildinfo" + "github.com/kaasops/vector-operator/internal/evcollector" ) func main() { @@ -112,7 +114,7 @@ func main() { http.Handle("/metrics", promhttp.Handler()) go func() { - if err = http.ListenAndServe(net.JoinHostPort("", *port), nil); err != nil && !errors.Is(http.ErrServerClosed, err) { + if err = http.ListenAndServe(net.JoinHostPort("", *port), nil); err != nil && !errors.Is(err, http.ErrServerClosed) { log.Error("failed to start http server", "error", err) os.Exit(1) } diff --git a/cmd/evgen/main.go b/cmd/evgen/main.go index 21c50a48..41c0ef94 100644 --- a/cmd/evgen/main.go +++ b/cmd/evgen/main.go @@ -4,11 +4,12 @@ import ( "context" "flag" "fmt" - "k8s.io/apimachinery/pkg/util/rand" - ctrl "sigs.k8s.io/controller-runtime" "sync" "time" + "k8s.io/apimachinery/pkg/util/rand" + ctrl "sigs.k8s.io/controller-runtime" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -34,8 +35,8 @@ func main() { wg := sync.WaitGroup{} for i := 0; i < *workers; i++ { + wg.Add(1) go func() { - wg.Add(1) defer wg.Done() clientset, err := kubernetes.NewForConfig(config) diff --git a/cmd/manager/main.go b/cmd/manager/main.go index 2f8ea057..61be4773 100644 --- a/cmd/manager/main.go +++ b/cmd/manager/main.go @@ -21,10 +21,11 @@ import ( "crypto/tls" "flag" "fmt" - "github.com/kaasops/vector-operator/internal/buildinfo" "os" "time" + "github.com/kaasops/vector-operator/internal/buildinfo" + monitorv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" diff --git a/docs/ci-cd.md b/docs/ci-cd.md new file mode 100644 index 00000000..5a4e3dbb --- /dev/null +++ b/docs/ci-cd.md @@ -0,0 +1,174 @@ +# CI/CD Documentation + +## GitHub Actions Workflows + +### E2E Tests Workflow + +The E2E tests workflow automatically runs end-to-end tests for every pull request and push to the main branch. + +**Workflow File:** `.github/workflows/e2e-tests.yaml` + +#### Triggers + +- **Push to main/master**: Runs on every push to the main or master branch +- **Pull Requests**: Runs on PRs targeting main/master +- **Manual**: Can be triggered manually via GitHub Actions UI (workflow_dispatch) + +#### Workflow Steps + +1. **Checkout code**: Clones the repository +2. **Set up Go**: Installs Go using version from `go.mod` +3. **Install dependencies**: Installs kubebuilder for CRD generation +4. **Create kind cluster**: Creates a single-node Kubernetes cluster using `scripts/kind-config-ci.yaml` +5. **Verify cluster**: Checks cluster health and connectivity +6. **Build image**: Builds operator Docker image +7. **Load image**: Loads image into the kind cluster +8. **Run E2E tests**: Executes `make test-e2e` with JUnit reporting +9. **Upload test results**: Saves test results as artifacts (retained for 7 days) +10. **Publish test results**: Publishes JUnit results as GitHub check + +#### Configuration + +**Kind Cluster (CI):** `scripts/kind-config-ci.yaml` +- Single control-plane node +- Control-plane allows scheduling workloads for faster execution +- Port mappings for ingress (80, 443) + +#### Test Reports + +Test results are available in multiple formats: + +1. **JUnit XML**: `test/e2e/results/run-*/reports/junit-report.xml` + - Machine-readable format + - Used by GitHub Actions to display test results + +2. **JSON Report**: `test/e2e/results/run-*/reports/report.json` + - Detailed test execution data + - Suitable for programmatic analysis + +3. **Plain text log**: `test/e2e/results/run-*/reports/test-output.log` + - Human-readable test output + - Contains full test execution logs + +4. **HTML Report**: Generated via `make test-report` + - Interactive visualization + - Requires Python 3 + +#### Artifacts + +**Test Results** (7 days retention): +- JUnit XML report +- JSON report +- Plain text test output +- Failure artifacts (pod logs, events, resource states) +- Available for all workflow runs + +#### Viewing Results + +1. **GitHub UI**: + - Go to Actions tab → E2E Tests workflow + - Click on a specific run to view results + +2. **PR Checks**: + - Test results appear as a check on PRs + - Click "Details" to view full report + +#### Running E2E Tests Locally + +```bash +# Run e2e tests with full reporting +make test-e2e + +# Run with fail-fast (stop on first failure) +make test-e2e E2E_FAIL_FAST=true + +# Run with label filter +make test-e2e E2E_LABEL_FILTER="smoke" + +# Run with description +make test-e2e E2E_RUN_DESCRIPTION="Testing new feature" + +# Generate HTML report from results +make test-report +``` + +#### Troubleshooting + +**Tests fail in CI but pass locally:** +- Check timing issues (CI may be slower) +- Verify kind-config-ci.yaml configuration +- Check resource limits in CI environment + +**Cluster creation timeout:** +- Increase `wait` timeout in workflow +- Check Docker daemon health in CI +- Verify kind version compatibility + +**Image loading fails:** +- Ensure Docker build succeeds +- Check image names match between build and load steps +- Verify kind cluster name is correct + +**Tests timeout:** +- Default timeout is 30 minutes +- Adjust `timeout-minutes` in workflow if needed +- Check for hanging pods or resources + +#### Manual Trigger + +To manually trigger the E2E tests workflow: + +1. Go to Actions tab in GitHub +2. Select "E2E Tests" workflow +3. Click "Run workflow" button +4. Select branch and click "Run workflow" + +#### Performance + +**Typical execution time:** +- Cluster creation: ~1-2 minutes +- Image build: ~2-3 minutes +- Image load: ~30 seconds +- E2E tests: ~10-15 minutes +- **Total: ~15-20 minutes** + +### Lint Workflow + +**Workflow File:** `.github/workflows/lint.yaml` + +#### Jobs + +1. **golangci-lint**: Runs golangci-lint with project configuration +2. **go fmt**: Checks code formatting +3. **go vet**: Runs Go static analysis + +#### Configuration + +Linter configuration is defined in `.golangci.yml`: + +```yaml +linters: + enable: + - gosimple + - govet + - ineffassign + - staticcheck + - unused + - errcheck + - gofmt + - goimports + +linters-settings: + goimports: + local-prefixes: github.com/kaasops/vector-operator +``` + +#### Running Locally + +```bash +# Run linter +make lint + +# Run linter with auto-fix +make lint-fix +``` diff --git a/docs/specification.md b/docs/specification.md index dea9e45c..5eb3000c 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -115,7 +115,7 @@ address - The network address to which the API should bind. If you’re running Vector in a Docker container, make sure to bind to 0.0.0.0. Otherwise the API will not be exposed outside the container. By default - 0.0.0.0:8686 + The network address to which the API should bind. Uses dual-stack IPv6/IPv4 binding (::) by default, which accepts connections on both IPv4 and IPv6. By default - [::]:8686 enabled diff --git a/go.mod b/go.mod index 2e47d2db..c618e4a8 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,14 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/go-logr/logr v1.4.2 github.com/mitchellh/mapstructure v1.5.0 - github.com/onsi/ginkgo/v2 v2.19.0 - github.com/onsi/gomega v1.33.1 + github.com/onsi/ginkgo/v2 v2.20.2 + github.com/onsi/gomega v1.34.1 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.60.1 github.com/prometheus/client_golang v1.19.1 github.com/spf13/viper v1.19.0 github.com/stoewer/go-strcase v1.2.0 github.com/stretchr/testify v1.9.0 - golang.org/x/sync v0.7.0 + golang.org/x/sync v0.8.0 google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 gopkg.in/yaml.v2 v2.4.0 @@ -58,7 +58,7 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect + github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -109,14 +109,14 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect - golang.org/x/net v0.26.0 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect + golang.org/x/net v0.28.0 // indirect golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/term v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/sys v0.24.0 // indirect + golang.org/x/term v0.23.0 // indirect + golang.org/x/text v0.17.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect diff --git a/go.sum b/go.sum index f749aa74..c6e55c2d 100644 --- a/go.sum +++ b/go.sum @@ -191,8 +191,8 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 h1:5iH8iuqE5apketRbSFBy+X1V0o+l+8NF1avt4HWl7cA= +github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -261,10 +261,10 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA= -github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= -github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= -github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4= +github.com/onsi/ginkgo/v2 v2.20.2/go.mod h1:K9gyxPIlb+aIvnZ8bd9Ak+YP18w3APlR+5coaZoE2ag= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -412,8 +412,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -467,8 +467,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -487,8 +487,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -528,19 +528,19 @@ golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= -golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.21.0 h1:WVXCp+/EBEHOj53Rvu+7KiT/iElMrO8ACK16SMZ3jaA= -golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -588,8 +588,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/helm/charts/vector-operator/Chart.yaml b/helm/charts/vector-operator/Chart.yaml index a560dc3d..beafa686 100644 --- a/helm/charts/vector-operator/Chart.yaml +++ b/helm/charts/vector-operator/Chart.yaml @@ -15,13 +15,13 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "0.7.1" +version: "0.7.2" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.3.2" +appVersion: "v0.3.3" home: https://github.com/kaasops/vector-operator sources: diff --git a/helm/index.yaml b/helm/index.yaml index 35cb2a4c..75d50fd5 100644 --- a/helm/index.yaml +++ b/helm/index.yaml @@ -1,9 +1,22 @@ apiVersion: v1 entries: vector-operator: + - apiVersion: v2 + appVersion: v0.3.3 + created: "2025-11-17T12:42:54.486235+02:00" + description: A Helm chart to install Vector Operator + digest: d1e04fd4039e06ce24d89feb9707a7f4a65f3fd4b2bec6f4f0d937b4c9775c4f + home: https://github.com/kaasops/vector-operator + name: vector-operator + sources: + - https://github.com/kaasops/vector-operator + type: application + urls: + - https://kaasops.github.io/vector-operator/helm/packages/vector-operator-0.7.2.tgz + version: 0.7.2 - apiVersion: v2 appVersion: v0.3.2 - created: "2025-10-03T11:33:29.251489168+03:00" + created: "2025-11-17T12:42:54.484496+02:00" description: A Helm chart to install Vector Operator digest: 94e6f3d7ad7f41a8edf03e72ffe2f2586f9d43d0762899025a274b1c2329088c home: https://github.com/kaasops/vector-operator @@ -16,7 +29,7 @@ entries: version: 0.7.1 - apiVersion: v2 appVersion: v0.3.2 - created: "2025-10-03T11:33:29.253969117+03:00" + created: "2025-11-17T12:42:54.487982+02:00" description: A Helm chart to install Vector Operator digest: 67fbdd5181070c542bc7b52457ff15962d6b1dcefe495939f076703f71cd0bde home: https://github.com/kaasops/vector-operator @@ -29,7 +42,7 @@ entries: version: "0.7" - apiVersion: v2 appVersion: v0.3.0 - created: "2025-10-03T11:33:29.249246225+03:00" + created: "2025-11-17T12:42:54.482541+02:00" description: A Helm chart to install Vector Operator digest: 69262a286e22bfbf7571297f05d17dfc8f19e6215faa42f4dbdabea8a5610586 home: https://github.com/kaasops/vector-operator @@ -42,7 +55,7 @@ entries: version: "0.6" - apiVersion: v2 appVersion: v0.2.0 - created: "2025-10-03T11:33:29.24645919+03:00" + created: "2025-11-17T12:42:54.480921+02:00" description: A Helm chart to install Vector Operator digest: f0e89cc2f3b641588e603107ba4aedc5f1ec585452c88bac46784226e56751e2 home: https://github.com/kaasops/vector-operator @@ -55,7 +68,7 @@ entries: version: "0.5" - apiVersion: v2 appVersion: v0.1.2 - created: "2025-10-03T11:33:29.244146612+03:00" + created: "2025-11-17T12:42:54.479082+02:00" description: A Helm chart to install Vector Operator digest: e1fe0e96c146c7c275c181e727c8a60f21898cabe90629851a2920d2915f84b7 home: https://github.com/kaasops/vector-operator @@ -68,7 +81,7 @@ entries: version: "0.4" - apiVersion: v2 appVersion: v0.1.1 - created: "2025-10-03T11:33:29.241579734+03:00" + created: "2025-11-17T12:42:54.477278+02:00" description: A Helm chart to install Vector Operator digest: a916c9e9f81bdbf9f734073fb453a6b67d7a724ed7ff4326d7884b136c103ce5 home: https://github.com/kaasops/vector-operator @@ -81,7 +94,7 @@ entries: version: "0.3" - apiVersion: v2 appVersion: v0.1.1 - created: "2025-10-03T11:33:29.23920403+03:00" + created: "2025-11-17T12:42:54.475585+02:00" description: A Helm chart to install Vector Operator digest: 582d95c6f63134f6cd815bcb85adce5770e179de21a979ec76f91ab7d8531b45 home: https://github.com/kaasops/vector-operator @@ -94,7 +107,7 @@ entries: version: "0.2" - apiVersion: v2 appVersion: v0.1.1 - created: "2025-10-03T11:33:29.237032712+03:00" + created: "2025-11-17T12:42:54.473719+02:00" description: A Helm chart to install Vector Operator digest: 3ac5a422f3f1861528f737a0fea077cad5f7b7516db3fe5d392c887d5d3459d5 home: https://github.com/kaasops/vector-operator @@ -107,7 +120,7 @@ entries: version: 0.1.1 - apiVersion: v2 appVersion: v0.1.0 - created: "2025-10-03T11:33:29.233939161+03:00" + created: "2025-11-17T12:42:54.471729+02:00" description: A Helm chart to install Vector Operator digest: 191ec4f83f11541df19680ce220992c84fb10210f0d54a5acc29361ccfd787bb home: https://github.com/kaasops/vector-operator @@ -120,7 +133,7 @@ entries: version: 0.1.0 - apiVersion: v2 appVersion: pre-v0.1.0-r1 - created: "2025-10-03T11:33:29.231243725+03:00" + created: "2025-11-17T12:42:54.469521+02:00" description: A Helm chart to install Vector Operator digest: 01bd2e347c5782127511a0a0fbbab72508cbe667664f2c9b615cabb80a4c40c7 home: https://github.com/kaasops/vector-operator @@ -133,7 +146,7 @@ entries: version: 0.1.0-rc1 - apiVersion: v2 appVersion: v0.0.40 - created: "2025-10-03T11:33:29.226307689+03:00" + created: "2025-11-17T12:42:54.46547+02:00" description: A Helm chart to install Vector Operator digest: c50e673e811b8d4c03ad45c92ce23b9bc54eef4665091e70fab9a4b7e4c6c3f1 home: https://github.com/kaasops/vector-operator @@ -146,7 +159,7 @@ entries: version: 0.0.40 - apiVersion: v2 appVersion: v0.0.39 - created: "2025-10-03T11:33:29.225419535+03:00" + created: "2025-11-17T12:42:54.464527+02:00" description: A Helm chart to install Vector Operator digest: e1de38c869bf896bfb9f5f615c329d3ccce3bd91cb64bb6fa1c783a40ea290a2 home: https://github.com/kaasops/vector-operator @@ -159,7 +172,7 @@ entries: version: 0.0.39 - apiVersion: v2 appVersion: v0.0.38 - created: "2025-10-03T11:33:29.224540967+03:00" + created: "2025-11-17T12:42:54.463849+02:00" description: A Helm chart to install Vector Operator digest: 565b148184400900f5572b06ceebaac6340af5e5fd1122b308bdbfcbc2d2040a home: https://github.com/kaasops/vector-operator @@ -172,7 +185,7 @@ entries: version: 0.0.38 - apiVersion: v2 appVersion: v0.0.37 - created: "2025-10-03T11:33:29.223624926+03:00" + created: "2025-11-17T12:42:54.463138+02:00" description: A Helm chart to install Vector Operator digest: 65e10ee46e6855ba95f51e21ca14abc4411191b260c6b96ec72c775e73c1e331 home: https://github.com/kaasops/vector-operator @@ -185,7 +198,7 @@ entries: version: 0.0.37 - apiVersion: v2 appVersion: v0.0.36 - created: "2025-10-03T11:33:29.222291666+03:00" + created: "2025-11-17T12:42:54.46229+02:00" description: A Helm chart to install Vector Operator digest: 972b6b4048b6d17b0616786e49915ef52cb7c9573cfb6eb359b6c19b66eabe31 home: https://github.com/kaasops/vector-operator @@ -198,7 +211,7 @@ entries: version: 0.0.36 - apiVersion: v2 appVersion: v0.0.35 - created: "2025-10-03T11:33:29.221443038+03:00" + created: "2025-11-17T12:42:54.461582+02:00" description: A Helm chart to install Vector Operator digest: d03ba759c42f2bd8d8f1df71702ee4f26a73b8bc28760ca7254af20d811cee8a home: https://github.com/kaasops/vector-operator @@ -211,7 +224,7 @@ entries: version: 0.0.35 - apiVersion: v2 appVersion: v0.0.34 - created: "2025-10-03T11:33:29.220522814+03:00" + created: "2025-11-17T12:42:54.460887+02:00" description: A Helm chart to install Vector Operator digest: 01e9d488ee78c8603d821f96344edee568ebcb42049d586de37b8df39b372bd4 home: https://github.com/kaasops/vector-operator @@ -224,7 +237,7 @@ entries: version: 0.0.34 - apiVersion: v2 appVersion: v0.0.33 - created: "2025-10-03T11:33:29.219572877+03:00" + created: "2025-11-17T12:42:54.460169+02:00" description: A Helm chart to install Vector Operator digest: fcde3c94a0fa6caa5f3d333226c95b7c85ede8489d46277e1222a868ed4ec8c3 home: https://github.com/kaasops/vector-operator @@ -237,7 +250,7 @@ entries: version: 0.0.33 - apiVersion: v2 appVersion: v0.0.32 - created: "2025-10-03T11:33:29.218359973+03:00" + created: "2025-11-17T12:42:54.458973+02:00" description: A Helm chart to install Vector Operator digest: 26323037ec47f1703ea930a99ab4ec8fb93b44975ce969514ea68d4130017015 home: https://github.com/kaasops/vector-operator @@ -250,7 +263,7 @@ entries: version: 0.0.32 - apiVersion: v2 appVersion: v0.0.31 - created: "2025-10-03T11:33:29.217283384+03:00" + created: "2025-11-17T12:42:54.457945+02:00" description: A Helm chart to install Vector Operator digest: 45b924c07a825e0f7cd3fb534a6ffd16604790d13be1aff59150c045474754e3 home: https://github.com/kaasops/vector-operator @@ -263,7 +276,7 @@ entries: version: 0.0.31 - apiVersion: v2 appVersion: v0.0.30 - created: "2025-10-03T11:33:29.216352085+03:00" + created: "2025-11-17T12:42:54.457232+02:00" description: A Helm chart to install Vector Operator digest: 03beda549d15f50325028ea29af5f2065ac0b8adf3078bf7dc1312981aa5e7db home: https://github.com/kaasops/vector-operator @@ -276,7 +289,7 @@ entries: version: 0.0.30 - apiVersion: v2 appVersion: v0.0.29 - created: "2025-10-03T11:33:29.21547395+03:00" + created: "2025-11-17T12:42:54.456151+02:00" description: A Helm chart to install Vector Operator digest: 0f025fc3a924b37b8c4131c4d8cfa437d2d4e557ab9476ed3e69a00232c7dca6 home: https://github.com/kaasops/vector-operator @@ -289,7 +302,7 @@ entries: version: 0.0.29 - apiVersion: v2 appVersion: v0.0.28 - created: "2025-10-03T11:33:29.214044879+03:00" + created: "2025-11-17T12:42:54.455461+02:00" description: A Helm chart to install Vector Operator digest: af856d41314313e04f15e7143409a9c564c6ca610b0d2eaec3112add8573e668 home: https://github.com/kaasops/vector-operator @@ -302,7 +315,7 @@ entries: version: 0.0.28 - apiVersion: v2 appVersion: v0.0.27 - created: "2025-10-03T11:33:29.212974647+03:00" + created: "2025-11-17T12:42:54.454738+02:00" description: A Helm chart to install Vector Operator digest: 631e2ff02bbd7f247cb486494fd2af60c57cc551066a6a3858226551bc1745a4 home: https://github.com/kaasops/vector-operator @@ -315,7 +328,7 @@ entries: version: 0.0.27 - apiVersion: v2 appVersion: v0.0.26 - created: "2025-10-03T11:33:29.212181212+03:00" + created: "2025-11-17T12:42:54.453874+02:00" description: A Helm chart to install Vector Operator digest: 760a2833f4c1a33466982419b079ff18d996331ebacc40cf93b0f55229cdb7db home: https://github.com/kaasops/vector-operator @@ -328,7 +341,7 @@ entries: version: 0.0.26 - apiVersion: v2 appVersion: v0.0.25 - created: "2025-10-03T11:33:29.211347663+03:00" + created: "2025-11-17T12:42:54.453002+02:00" description: A Helm chart to install Vector Operator digest: fd22b996b071b6d85740ccf76e85cb640fa717c2620748d206d3f4fdd44cbcc2 home: https://github.com/kaasops/vector-operator @@ -341,7 +354,7 @@ entries: version: 0.0.25 - apiVersion: v2 appVersion: v0.0.24 - created: "2025-10-03T11:33:29.210075092+03:00" + created: "2025-11-17T12:42:54.452119+02:00" description: A Helm chart to install Vector Operator digest: ea257e60ecde063a0d1ed52ce5e3283245b8f0e2daba58ea3a5adb0ba82d7799 home: https://github.com/kaasops/vector-operator @@ -354,7 +367,7 @@ entries: version: 0.0.24 - apiVersion: v2 appVersion: v0.0.23 - created: "2025-10-03T11:33:29.20912157+03:00" + created: "2025-11-17T12:42:54.451084+02:00" description: A Helm chart to install Vector Operator digest: 546d202b3b9263f789b88335263191098dfcabd5d8698105f37cad24d56a8ed0 home: https://github.com/kaasops/vector-operator @@ -367,7 +380,7 @@ entries: version: 0.0.23 - apiVersion: v2 appVersion: v0.0.22 - created: "2025-10-03T11:33:29.208283787+03:00" + created: "2025-11-17T12:42:54.450342+02:00" description: A Helm chart to install Vector Operator digest: bf96ddc8ac61e9d6beb8bc763fbf3fa6025d950b29d70d80de6e8a0ea45e0411 home: https://github.com/kaasops/vector-operator @@ -380,7 +393,7 @@ entries: version: 0.0.22 - apiVersion: v2 appVersion: v0.0.21 - created: "2025-10-03T11:33:29.207029762+03:00" + created: "2025-11-17T12:42:54.44935+02:00" description: A Helm chart to install Vector Operator digest: d37b3064c0374d71e06c0131bcac2bf9e60ec4d62fcbbb20704c5277eabd899d home: https://github.com/kaasops/vector-operator @@ -393,7 +406,7 @@ entries: version: 0.0.21 - apiVersion: v2 appVersion: v0.0.20 - created: "2025-10-03T11:33:29.206084826+03:00" + created: "2025-11-17T12:42:54.448068+02:00" description: A Helm chart to install Vector Operator digest: b95cd9ea8b74fde85175411129f77bf7a7afb4e9324ba2d02d489d0d6ef42d6d home: https://github.com/kaasops/vector-operator @@ -406,7 +419,7 @@ entries: version: 0.0.20 - apiVersion: v2 appVersion: v0.0.19 - created: "2025-10-03T11:33:29.205569011+03:00" + created: "2025-11-17T12:42:54.447643+02:00" description: A Helm chart to install Vector Operator digest: bc1acd8b21a95e373702daa9c4ce4226b28f56b9c9299482d47b200baddbec14 home: https://github.com/kaasops/vector-operator @@ -419,7 +432,7 @@ entries: version: 0.0.19 - apiVersion: v2 appVersion: v0.0.18 - created: "2025-10-03T11:33:29.205016352+03:00" + created: "2025-11-17T12:42:54.447209+02:00" description: A Helm chart to install Vector Operator digest: 2bf9cde6eec7b00bfc70d7ac79b1e9d4bf3a406749c6b2bd816f20efd0cb44c3 home: https://github.com/kaasops/vector-operator @@ -432,7 +445,7 @@ entries: version: 0.0.18 - apiVersion: v2 appVersion: v0.0.17 - created: "2025-10-03T11:33:29.204483226+03:00" + created: "2025-11-17T12:42:54.44676+02:00" description: A Helm chart to install Vector Operator digest: edb51a059b9231f9bc2e2e0dd82c432d0e799a6767a7829ee113054478e098ed home: https://github.com/kaasops/vector-operator @@ -445,7 +458,7 @@ entries: version: 0.0.17 - apiVersion: v2 appVersion: v0.0.16 - created: "2025-10-03T11:33:29.203627979+03:00" + created: "2025-11-17T12:42:54.445975+02:00" description: A Helm chart to install Vector Operator digest: 06e33602d72c44cf6779152df4936133ed87e228dd71cbb6615aa4c2666a1ee1 home: https://github.com/kaasops/vector-operator @@ -458,7 +471,7 @@ entries: version: 0.0.16 - apiVersion: v2 appVersion: v0.0.15 - created: "2025-10-03T11:33:29.203008445+03:00" + created: "2025-11-17T12:42:54.44548+02:00" description: A Helm chart to install Vector Operator digest: 6c9f5ba7a914329caa4f93342d3415fcf4e5fe39f5b7db69173896ea13a47c5b home: https://github.com/kaasops/vector-operator @@ -471,7 +484,7 @@ entries: version: 0.0.15 - apiVersion: v2 appVersion: v0.0.14 - created: "2025-10-03T11:33:29.20241002+03:00" + created: "2025-11-17T12:42:54.44504+02:00" description: A Helm chart to install Vector Operator digest: 9f7a3b66247dea7f826b2b38202b0ddfa72b30ecc0954d75be36e066deda9df9 home: https://github.com/kaasops/vector-operator @@ -484,7 +497,7 @@ entries: version: 0.0.14 - apiVersion: v2 appVersion: v0.0.13 - created: "2025-10-03T11:33:29.201794885+03:00" + created: "2025-11-17T12:42:54.444616+02:00" description: A Helm chart to install Vector Operator digest: c88a1866a20fb2aea4a23886e6e60080eba9ae7ef2706f492d9b329dc9ddf49b home: https://github.com/kaasops/vector-operator @@ -497,7 +510,7 @@ entries: version: 0.0.13 - apiVersion: v2 appVersion: v0.0.12 - created: "2025-10-03T11:33:29.201258232+03:00" + created: "2025-11-17T12:42:54.443996+02:00" description: A Helm chart to install Vector Operator digest: 384e8fd8f8f743036eaf1415d893158256a2ad9daddcb17a3d0701a528d9f0df home: https://github.com/kaasops/vector-operator @@ -510,7 +523,7 @@ entries: version: 0.0.12 - apiVersion: v2 appVersion: v0.0.11 - created: "2025-10-03T11:33:29.200722312+03:00" + created: "2025-11-17T12:42:54.442767+02:00" description: A Helm chart to install Vector Operator digest: 29e1e04c1706b88ef61ed6c91a45847e6069843419515a33046c5929b179e273 home: https://github.com/kaasops/vector-operator @@ -523,7 +536,7 @@ entries: version: 0.0.11 - apiVersion: v2 appVersion: v0.0.10 - created: "2025-10-03T11:33:29.199843802+03:00" + created: "2025-11-17T12:42:54.442243+02:00" description: A Helm chart to install Vector Operator digest: f4398224ce88b852b319c950d0f39bfd5e6181801c1fac1b42b069dd2d358078 home: https://github.com/kaasops/vector-operator @@ -536,7 +549,7 @@ entries: version: 0.0.10 - apiVersion: v2 appVersion: v0.0.9 - created: "2025-10-03T11:33:29.228477933+03:00" + created: "2025-11-17T12:42:54.467253+02:00" description: A Helm chart to install Vector Operator digest: 66c528b6daa9f6fb9a8dd91895b69151f3f0183f4685ba4a2bc026fac27f25a7 home: https://github.com/kaasops/vector-operator @@ -549,7 +562,7 @@ entries: version: 0.0.9 - apiVersion: v2 appVersion: v0.0.8 - created: "2025-10-03T11:33:29.228015585+03:00" + created: "2025-11-17T12:42:54.46693+02:00" description: A Helm chart to install Vector Operator digest: 21c4c214cd0206abb743e82ac757804d644de08d80eb5f2edbb82ff9668cfed3 home: https://github.com/kaasops/vector-operator @@ -562,7 +575,7 @@ entries: version: 0.0.8 - apiVersion: v2 appVersion: v0.0.7 - created: "2025-10-03T11:33:29.227394919+03:00" + created: "2025-11-17T12:42:54.466432+02:00" description: A Helm chart to install Vector Operator digest: 27915a2bf70da3f66d08cf4a1f6c41ad38937759ad52eaf8b19f5a3e348e2f2e home: https://github.com/kaasops/vector-operator @@ -575,7 +588,7 @@ entries: version: 0.0.7 - apiVersion: v2 appVersion: v0.0.6 - created: "2025-10-03T11:33:29.226752791+03:00" + created: "2025-11-17T12:42:54.465943+02:00" description: A Helm chart to install Vector Operator digest: 26760fbc2018336c12e8726307a624970ee994c4ffa021cc216c13669bd82f09 home: https://github.com/kaasops/vector-operator @@ -588,7 +601,7 @@ entries: version: 0.0.6 - apiVersion: v2 appVersion: v0.0.5 - created: "2025-10-03T11:33:29.199278578+03:00" + created: "2025-11-17T12:42:54.441691+02:00" description: A Helm chart to install Vector Operator digest: 1d6034027ae2f08a9dbea4d6ee9a1604117ae44d9daceb3f654b87a99175251f home: https://github.com/kaasops/vector-operator @@ -599,4 +612,4 @@ entries: urls: - https://kaasops.github.io/vector-operator/helm/packages/vector-operator-0.0.1.tgz version: 0.0.1 -generated: "2025-10-03T11:33:29.198717087+03:00" +generated: "2025-11-17T12:42:54.440609+02:00" diff --git a/helm/packages/vector-operator-0.7.2.tgz b/helm/packages/vector-operator-0.7.2.tgz new file mode 100644 index 0000000000000000000000000000000000000000..e3868d96f085de7d7af6d51c798a8853d0d9d197 GIT binary patch literal 102637 zcmafab95&`({5}V8{4*R+qS*2ZEIuOc6Q@m>||qG8{>ELzTbE6zjw}=)7@3w(=}65 zeV&?mnkW_q1LQvkkOqX#L|T>EOj?0c-kXQRlueV_T#dsF3vqnz4C7@*x(DMtY{3htKO0?RwpX?mwBM`(K+C4fp^E znO0VG8xR@HJ^)t(O9Ph@D-mhX-z+*A6{k;FnXytYF~X{!}1Y z#pZ>mzf4;B{f;7+V-0_2V(DDgfR(BXYF_flg8<&xdO^hj_5rr9pj!T6Q^d{8#Ev!! z+N?6{e8{b|4IadyKoY9A-GsFhqo}9kvF_OU<)QiLBY$?B0BZhUm1^-?MOpU_Y?TMm zS3L@}GZv3_u5-0a!_mvQIc}s*DTRS&EJyi@gND)BXF}hF0%e*TAS5DMdHWQI?i28Y zDL+tS8))RmaS6;zvq6Kfyy-Jwrsvm@tkth$3IGI$Yl-L_Ukl zyFw+^p~qMiWt^4X;-nB?%&v;^6D9A-)Mt~p{oqkrpCcr3L4#N&-QB*A(^uGm@n^_3 zP2S#o4~vrwlW<4N;&L)XQ(|4a4U-gvwGp?XH%}%!NPX6yjm~$IfM{$n8?G>?K_k_n z#zF`LTk(`fP(s|*ofPsPKZc- z|3rg`5o%?QmRI9cD!tEkfBEkp`}=)-lw;VOXX!0R$=4>ec22{3!%Pbl>>3dKhj@ah`y&&$>w)dafnoFzw*F6u&~KkCCe^pQEsKkdw};pj z4PDt*UAu&Q&-`nUb5omH-hPva^MLGpHW})sS=1P0G!AT;UHlNyx4Y-4&$Qg02Wm9D zF;hjo^Y>#R0js0)Czzu3mNV0VTsG4uM%2aX4ta}cQ?s8>wz8x}(WZEw$WP(UkbOLq z7o*P-4zGrM(Jc!ng>lXaE9Xx4hk znuWS6HgXeH>tCqM9U_wZTz0ZEEBNQ+MQcE{Hfq6oC;Uh$fu{MIHJD#je<&5oj5=1t zDCPF2v%OBR!bb027_*siS*Dk=V$sbbzpR_dgR_}d+3#nQ&GV!937xYNQ4_J)cfY8Q zS>S*EStzr5unNKv4q2?_glS^YfAB>D!kG+y)SwcEw!8LGD-=;pWZ9Y z<`-0dy2~+K!9lspP}~Y;69<c>CSWKQx&mc9HjRzCFyX2$S?Oc_rcbHDI zoHrG+@FGrLCsO9!guYGy5Gmq3v%*&{{KfZ9d1dbUQ;`FzxzKWw0bdw7HlUu*>k|zF*Vz?Mj6#1Tndd07!61(B^8Fpf{Wy_F=JcyUlBwjN7mR0BN(q7rJa{$rQ(lf$^<8_lX)X#dO08Gvg2nHkuiq6+$QU7TM2? zWTRBpc-69Ss&BAQe_|m&YnB}RXs6vO$LlW$tp9n0eQ!p}PS!QAxsoz5uAN+xt>^1h z{foit#xc9$o>(efKsedjBgVNVv8D%6cWvBkL=oJBUBxatRci<7Vk~vwrN7EdVq|Re zN!NyuegE&QP{qeVBl~{bP<-OKZS1|6d+vbxP5)=(PF2b>!Pgqqzuy@?ud}ki|Lzh|Oxzk;i+yKx~h zY-u(wQe$6|MT~}mYs1fn*(#q()6`U>K!PLdnSRLbG>!fUTCBCpagW){cA|7#tE7CI zfutxY!C3a!zYbn=3K*mLi&X0^va!Ho6Ofyp)z~o@&K_M2#h$Rlqh$pfFSljv?#O`+ zN^`VA%(&4X|C*JD#`W`HX->JM{!~NlvQ_Je;vEz;-L@c81!t{e2NCGjH{9>+5uNd_ z356pUV96`m=Ss8OqAnlYx{R`a5IKhvnO67l_{l2gItYeP+Id*FEw%IR|5&cP$3`c7 z<=*(AIbGWXi?p^_po$GYXI(m&$$yQgr z!h_js-n@015Eh|tk)7=4*Q{emu1{$D88Cm~r^*V@|7(O8cvX7X!4hwl9_`9QRWnJT z+fYA7&|>CNms%>RE78UvYm;EYQ-(YfiVo{m-@;+w;#isi%2TlrkVpJyJ@0m8PT$e! zh_cUhLkrPW2j587(-t?pj#G-H93mXcr{j)Dk<#5wn?=AoQ|pAYHp+YX(NQyz4d ze1-90?~N-yLO-rm!0F+t(X6ZHkOscXorHpwPOOHX=#2Uos5>%6sfGTm;#3<}gysn3 zd3mde>Z+3!4wS@K%8@R`b0_F^vxmN=atE$;V(B% zrNV#0bg}v4`J-mQ zYa`lZ4Nq^r-J}le=Op>=i$Y8U2|*wyo{*m8GF$0q9OZqeh(%gb_HJZD}`S^e1e z^|$_FSwB8zs-cb~|EdPBm*Bp;kEY}~t1wsb_ezvCG!?o7wEK2oitRaTz^b73OIF(Q z{;}s^Ak|jbw^n8XYl-HuM29y20gVKhE`a?X=MPq)A04q^^&h13@*yii9szBfpyVOF zBElAUN&o6?>2P%Fyg=1RsmyT*_3JLjWYsi)Vs4cDCflR`#=@>loLtTN`6vq*0Z9fO z0Q79rc%@(peVXQm*28)G9e&3^%7X1=qrednhH)nI`=b;3dsp2q;ihh@wwD4pzvZ?= zljWYD8cv^UcAvRVf!hqu=$v!Mz`Om{OQ6f!zWW-Gk7jG@@hUVX9G;R%97k9h`2uY=<0$trc>x-5HS5uDR*t^c|OVa@Mcbpq6k*akyWt^tmVATFdi)bcPPFyZ@Rzz6&-*3N2$pHmcAg$di#YaV(rKw%AJ&PKCqKSE zHfA{SDC-SOJfVkqrAJF~sEc3u1e!tuZ(>Ql?6{85T@~6a(VASH_a;7nNtmbH4xY@% z{aNZMVJjc?ElBcmtB;c1*Osn}%4s1cZV{HBYY38RF(cIx<}hLoo15sbNFI2Gjb1}& z(m^ttDE|lrUw9Kr7(H~rppTB)Ram4zW0M&ivUU2ne~Q#&Apg#*)=m$fu{-{ps9wce z=@$$RtbO*~9a4FN0lS%wThJd8-_;n3Q)`Q1AFoqz8j+W_t;{@#Zu~j(=X&~PM|s%` z!Vg>eX1{Lwl*WB&yQDF?e7vikMl~loxsC-(s9riTfE)NsX;glm_lIc$m}R{{fwP)a zUQVExL8Ej_+FUEJK;ZrSp`o949he{Z`g#^?0=y|sb0TCv*7>Bm z9|b;W`BiF!dM*CKLKrU!sJbXelFXab@NkMy-@(SL%R#keZe| zk6P4^{Gb?WH$mobeISC?{Ysb|NP}viCLly+*1>8P8UKg&B4|A6bpDCr#B z=3-wq$$V)S_x3NA=i%BhOZ2e6w)&ZUH=w9F8U?gOMNz9@GM1TS`&W3?MOKuKdw~Dr z*NDTOohY1eD+e9uPY*ZATLX|Uus`wTkAtoO{^?(sa=(%DSXeynCfa4LH&q7$a4`cy zee%E{?$8@UJq5Czq!Muu^id~F!#e($smjB~(@$=8s&mM;Pl~|$km@p5fkHT7-4!f! zr-$~3#^zB9v!EiP=prZ`n@*D$piRibS2hEH=ag21}}_iNAP{_oA8QN$WGI z2BBZbgNa277_^;>U+l%pxK47sd6O^0M*Sebql1Fu#DJ+wL59{?H1HgaT|~xQ&)S47 z4T$r~dYDKodXSOyUT)s+j2r+x6cz2X%lUL~mkTq?R4Fy#+k+VSGMFMotk|ln1X*($odk8ArbVQg;7_XzuQ=AiX|;kdFwRS9oT`) zicZ&&0PI#*8$0e{2SD%e%tQA6!X=^t2brmE{#HcjV7l#>F!R z>RTAOqAux$Oz@0B9lx(+q|tH=998^X5ff2SIQ`>mBHqM`+}nw-jX4mte^^Lb8WSi- z*oA}0PQKXJFSG0&V2-9TMFikH9(UD;#o0hsP9eonOSzw(ffqs?&av!8yfHlRQ&uG^ zNB^SDJj!Ceicd_yr+pH~p$cNPlH}Jxs~;l3M}nnE!JUDJx3}6VLYjhC^ptd^YaDc4 zACvj36daJ6aaOJp>L5b3JW~9!sHVR4CjkSfMBK6_N z;E+#tw7dDNU!i15k)veE?GALT1gfMpHAQ!xj;Hx-TgJ&_IR$h?EYggOlVH37@|CK? z2kawEPUR@{612Z<9PqRYhETCnQc5h0^%p5PV|foA+~f^b2L@PIWTe#xh_Ww7t3 z_-Ke!b1Q4=(S0YyEW!YAq}r?}U1NIZ8>O*O>oUo zJJ#gWT6OdkbanvN1WRl;KSESm3WFRTfDz5|8q7Pf=sGTI&oeFSKHiLSqNH#rNZZEf zu=r3sGuk{%!p!7~Gpu~Jt!4W`cJjapDG`%l4xJXskSDQPxX+!Gu;& z864ExeGnz5A2X3E$6c(f73bqC#S-Ivip$*DJ)19*5y&2X&8zDgCF0fry%**V+zP~Z zeuzN}N+N3_))yjcxM6uBYiGJpCXookGsKtf)S->389POdizQa!o}zeOPEa1IeM@FL zC7zA7$ZprNHRsDK(UW9}z2Fx3iOe*ZVd?z%0(=4vn;dE+V%4(%#(=xtMZm`Ab}@nz z*}RJ4)Y=QqjC<;I$&`;*(Lh^ez2Xkg`IyWXs`M!=q^@{$27FkQoEwIZJJFcblDp_X(hIJjesIK1hIXCSkxpgpo(q;@~ zK-(#gSnSekZl;47g=fHav^IokJAet3nXnOMy`4s7;?FiHLQ^kI-IejNvqk4?N5~ER@A_K1F=ln)W}>zQPVYv~jUz??3J!pbu?)zySO( z7;8C}-oW;5t^Tl|zv}=<1G}uNII+HTL7AFv@%Igsb*`dSi21Rxb>=yK8_P?qT$fl! z<-C~Pr$*sKVB>s2cDodv$mfh9FRR2Cyr3|#{h1QR-M2W;wT{iqRtsN>B9-uekx%fz znRyB;u|iPMT?l^Jje~J^?{9Nt19o$lkE($;jLmtRzXPaPQpw1y`6Ok8FDgiKsAW=U%3 z2|?CJNnyHPJF4C+V}Mr_lfxu5MWFSS^&pmWpdCLeravz;)|%;&9mz0gh2-E=M1*8t z(z|J$*U-ilsJD+9buG-lM3l{GP3H1F;aZmp>oNUSW_B=`F;sOH-(YSGbR zXz2GDn?jU#Ghhcwnzb9o{<}?_v{*0wuc72ZMIuY#R*#enyx$>p!>ailaiY zj`z|{4_PqH;^Q|X`0Ubs+IupPVrokwNiR}&eBP*=3Gy(`>M}|D+|^140%HEHswB5j zy1*}LCBrn<9ja8k6{OkCt##MZa&cmah~nXMKDH#Y8*>3=O()A(v7u^wW`LHM++DUl z>;$=#hz5+-#^%KToLaT?7h{`$dCOm)h!(AdU>ntse-Y?i6M4nVdQr@NJT(;ir&Vd< zXwwEoQ){RAzc>u&vR_fT)|cg7eZ0Kcs;L%6IjuJjaCDPh^)a^Yml|oiyD=z0@+-#| zwOTHtRTk}vbl{JMp(@_@LQ?J^d~16v>cPkiL2=#IAYGL+xsI*9)Yi z<-+eXsoMGTVJ#9vY+| z0o}FpLxikrVo<*0#>lvrBwgS2^C@HGvxj|SWE=WWfj=4l)dvVYC0)MVDSIt`Sj=_b zm}VeWcV$98mrZ=`E?zTLzrr8CtuB3dcpW|M9AxPFQYkpq(X~1G8GOc58?=tRzqD^Y znh~S~oc#E37|w&8G6nNxoHQ>g&|i0_MvN1qhpqKD#mSZ_%Sx6Zx^!mp6;SYUH=j@yH6n}Az8mZAike26@(W|G1<;?nWVK}G% z_B9BrqH*F~)T4-hW$|eM|1!p~YbI_2Apag^u%e6PRJN6*6lz&32QKwT!om6!Iv3`N zUOG%2F8}_!Y*5h%uAU>sW#BL0$lg|7ZKzAzoSl6hU9xyNPX9PZSw$7E*kXphlsZ!7 z4YRli0;$Fbjv>#*QrbgfnhcOkv|&9LndcM1KR*H4&TRKYrEOtL zyct7EUhm3%zBcG3n(XtfAYnl!M0!yF)x{?(EHKk>4gCTY>2=eXxvlTSwZ}-rad3fq z%zE3OciZm})^o49a+66E7V$+XByzHHxXMGc zlR**BQcsxVzL;| zWHoNm$N-IX7&OIiT!W%-H$+6%wl~4+>@*S{~m`8PYO~{WEb^ zB5TyMC23??c$;ZZ%=m>Rl4Z-mLb(dEe0tdT!t6L};Z_t-JUMv>&D9zTt619eGS}g@ zQdb$#@+u_`gR++L^|;sbk5;zY=W)!Lh6ZVQ3dDGF%gP^bh=dnQi=BEl-M}3_&0_ZW zURkI2u9YTUJ|3Oj=%bdAXeNFSuxvc-!zOgRhtzcJTn=Why&{BZW`FUd+FL>>TmWiuC?ROGzgL_n+uF|Vy>^{g@a|tS$>Mg?KHz;m zq!4@WqT*_T?>W%5;eN=Bm-v!F`S-Zl_k+h^ug)1)+5B20H(`~h9>m4#xZA)J_o1lu zAyJ$AkExScLEx{SZnjbRG9Jooh39*w9gNkmwajVo-(O6?dJc^97@ITq2G6_Lte@@G zt0>NFb7Vkm5EgJ`73`&&kR?2;w z_|Y%c@-uhkJKMq1rB|CV@)c?0=NrXU9Eu}@+mb78kiOGHTqCRbPS<$954oL7AW1E z)_y46P(?L;2%@Vwiydl*yN*#Qr2y6er^vWVb^{5>V$}R!6^OgC^}+ZuKBiJBgfGdZ z*k4xVWe)zWn35^~iel^8dpOG+;WbnK2}To09{eVjOMw>8)bDxT8lsrs1xZx_)mkW) zYF|eng3)W9Z7TuL@&2lTRPAYE=dbT825U9P_B_znSK$xvT^u>;Y@u#6ByCE8zL@N! zz0|~HTw*AUt3Ty|f8RgLZQs_}>HS~@djPx0b334<&0gb+BdO#n^B@8!d_sS$v)7bBg0Nnkl-c4zY7-%pXhgz;KM6ObcGTolt@i+ zG|hgNpqZo|$({0~mKTe8ETIsvh-V|JSc;{sClmvY)sI{}E<$HyVK)R*ve?90GipJI z$jFd>R5^X(Wd&~WK$BysqYN@v-mn$?kxxj%fy{P|RZ#n3U{~fjs+)g-$*uWA)9Kx8 z+gg`fY0|_zFDx?+%@b2{g$^{A?qYk3kdY*p4mmArI^Dya^+H!s#*2B)FcTj7$0jC5 zyVg;k+IZ?CqXoV#G9i)nO1gtJ+{DI)Xmi47k~?Vrl>wQK23LKZFnO;2#;OK+O8i91 zB~sx^dL&e}hehbJwp5pNdovavRI|sCibg^_!B1ghJ*}0M%Vdu>Uvsv5eWNL<_KGty zJDYcDr+6YAr3x&JwN4KJ)Paf zsBd3$c5}`@#f+25?$)l@L+|=a`ph)C#60=g^gOBzSBq!Za((YgakctO_Vs)^x%4_pba#tJy0)~TdiJ$+sGbF5 zE!xbX`lUVu&$9oD;2ID;%M`!6d>8G_T?SRYAyQf^Wm5)Je(n}p(L@I8mSE%V*MwxK zrE~)-7HWxM$@R5zZ;1uMtOWnas8ZaIP|;<(n*&T`v^W)*&tzu_7Ns~ZbAC3jr6*X; z`k<<-Cm+RV<5x~<^X;uEf^<3Q7LpU`$L7b zpf9YY|1_>}ya*LFq>N(^8^=|o-MWs@qY(c1@b14TQ20M6um>a!vT~)H3$<}WDWN|^ zMP}EgZa|dy&Fq>S=++~>v;pBLgPoS6w=IuhTXISWm%L5lbg>sNXkNJq9?>NL$86mP z%gRCbgi)?HofvWE!S(hu-*t&{aW0dEFQnO6L;JALTfS14g(XA^s3FKf^D>eAux4%` z9I}5Yoo^tV)aMS8^PmCmT(XSH{{#f)W`C^|HhJhQljXb<&M!T$S@!3r4XF9Ge1Rfd z%x%{X|MA3Yy2Tj&O-LcX=OT(h$jWbWvS-W(3+XF61S3$|kbhz#L}DTaYEqZf(8hgy zE!rpOQ{E*RwK~Y>g$ouT0U4=xH7D3$w$MS3=Vjgd)1gsLh~zS0Wcf=~hT&@rx-$bax|x{=?nL$6I{ zk-{dDolk30`8Pl+vwB(JCI4g}&d3)XUm%O5bHJ)@jb|<+Fg4Y46jzmc_kzXhL5kJL3Bae(1oN7ZHk0AJ#tZm!}*>e-)d;atm=Co%7Z#GF} zCB{b@k3jx6nZ#1l|3Qh?iLkE6n_iLDXEC0|ap+J*FE(167_^6k+tS_sk@oXCiN$WW z7x6zzb3pZflsT;b2^!V@?`Uzwy_5|#i2iTfES(&E$B_QFCh@-=7Qp+yEwg3%9zkNJ zX;Jdua^-u{YP#=g`?v1Rz#Kp;uIB&CrsWOUWIT`Ka7LF#?D~6x!x=nRE5xprW>#Yf zB<*Ami@}(%@9Obn5T=e1RPDGR)3-GKTiW`4{%9l(6w*%q9!QlAWjPM2rk5{^lttpT za>M;TmBe=e%Q{WsrK#pk>*qF}_tEEt3`gUyr zHngv_*|&e&5uU!?egEDRiO|&&@mHt)Lo$u=|BZed8k@>nz6UvNzH`(S5bzhTUE!d) ze0`|3@%5Gae-IMqY*U(9HuepJMfic;qq1Em@pR$rluB+`(3+AA4~0QID1u(C#N>h_ zUACOU3SP4Emor0^eoB(fxtc=FS&K!vf4wk;k%0Hhy?9j6JS`dWH*Zm6U?V=+FdJ?> z>Segc4DEs~wZxkp+!xL}vG>w%w3s*LSPd=uN3I-XuQ*FO-p~IKzT)V!)IG3juw-HJ z96^z#lAw7&1swilw+5B}GJto(vr$>Rgp?l~`Q_~uA(Aj*2U^6T=JLNBP4dkvGe$+f zy()Aial+U(UQ~EW>|a|2f8o8yO4i3wUSMeC@{A*qR*Ht$4~@UqJ(qOup}YzEi}fXA zYqtZE)$`89q-0ApcGvtc_F^_p*&aSwJyj}&(ki!B9@(>bRqt+}=;}T4t8-1J!d@B8 z3&tgVVFk`OI?QM}7ZOGJT~q}7%Yfm`Ty`np`YM{O^`HftA2%Gmxp?GcqM<*x?NjVC za10sA@LXq;0p1|2NnJ39)9F`zGEPPO2PSKQrP?BQ=%!2LQ;VcN)dvVrWDz9P#SpyXVWkiQ`w_s`U@H@6K&NWd9J;Cxd`Ny&iU#$45~>aBEQ z*pEHwnWzkzPdv|^&j#bGOL%lUY@{=2LQ0aNB57sJ3$%iAQQ=HJ7o|H_Go6~n{^q~N z#H)!Z`d2J%qK*8v6xw5*xzta#Ic9%Zl-NbJ8C{W|I3)xSoWI9m{}%qlvGQ*f47>~-%W~?uOTHS~ z+UvQrX~|e$;!8gdo?O)y(1=(4sio>~MZzfzHqj0Fj9o`#&ayJ>F?zdbEd#K(t)K*3 zw*Ks$!JS6~;r`0PJ4{%2pXC3vN}-NqOAb*i_2_{U*v(QMTWBFlr7~km3dzqylTAAw) zjy=N#7VLqiqQ-l7kJV5q*z)3VOd7tpr(b$o69?mV+%kH`?rQ#^uj_?&@yp`qJ#Fb1 zJd97)M@7$NBs!sqRHIHu`&_eL_;{qyX7`*;LeGoAWAfM2z0i|d29=Bwh-k8;L9 zd241$CbvkWfaTu0#t=-lcL8L-L|Oi|BsUa_T34vG46UlG>6imO+$Q5{Pc*fvu1pAZ zw3bvWQO@oKFWvq)SEp<$(t)0sQVW8ebh;TP>R&kWuCbrPVw&5Mk({4`s@`UDrsp_1 zlROCzkpU{%VSBJrU5g}Ji)Qz$OCJ5}60vqx{i^ardcHZE&Yq8~nk<+L`co4{OuGai zF}!%Gz>!kxMoHsuQovsH<*`DE1UdK=sm5X257H+LWU*tnYMn-YQApyJQw;_86!5h7 z`v*-QTlChV+)dKrXGTZzT=`by^hyd2&-NGkC7)9Qt6p*|YfpZzva*HZCj+)L5F zdid8v399;S^~HM@7|c%E#Fev$8f`39MS&yFPUq7w=`oABz1T{3qALA@7+Z^4)lL~% zIjs*{)K{Qw_ZPN@9i{O*%IE_Q41QkgK$6xQ1Be^WI$xrd$GMFs=#|#z#bz%w4qofk zclq1jw#!87CH=vieZS>9Z07Mjmhkf=%@OOo#hl)3rgGVj{Ff5qX~!JTM-{6_sVXe> z##n>az~6_6T$oJNbM$9XO!9>kcJODW5^^QCixI>D8 zC{6D~^7t|G;5n~(A^KKlP4YH$k)MOd)N>ZC? z51)z?DRV{4F7AP^y{7@kK<>cTw{wN*5~nar9%p;_{eHL%&}aOjQQ}fJiEHn}*!LV< z>xbAf7e7WZl7_OZdt)->bybaPMi<8cN>WMROsr!?4VO@8IUGm!dcbgw?K$^Th|Gk) z?``2w?NTsr%FiTk#{pF6;gTVJ>Mic&t%{!NnO`O`-LT|^ZueU=!HtkO{^dELbdF+d z#$>H+JZjNHO>|qG{s8@; za>>crkv?KlxK|{gw(aK8&7t2}gOM%vM^}Ka?_*#(R%1eOqk9Wu{gLH53EX6o##nT( zNs%^!nhsk!!zBr3uHAV1!4_Ziy{K(kO-2~v#~U7`MQ`un_JlUfUv~gm@^MyJg>hV_ zXz6_i4_~+QFB-%p$92ivLAz`C1_`v~%9)-z5nb3j9dgy44xX(Mi~x#Cu%_mEa148^ z)pX&&Yj+86P}33GSGRK1?3H~zX-R*SRMYB{(-u@!k1@ZG;(jB2izn8`%ub%4RuNZm z#?B_cf2m=K-k@p@cs}P24aA2W#iP905i+-)IEljUY{I+W(Smyk!cE5;H=EqZFJkGn zBN4L;cIA5K9t?&W(EX#m1tM4}eLu!?QE)a4RaKA&l zZgO^I(msB14PFw{h7yJDQeMp-RlOk!QpBd3dD1FL4zvvDq9D(cbWR7Jxp92dY{Yhz zR)?9Pg>Fboc2F(lnxLW}T*ys2l}FFz8X9X~XWV`TN_u0>txjexsuR;Nl|JxB;oQFqn4gzv_aG(ZRrHCGN+P)o=fQw_dI@xrWR@Rd zKV0^w4S-!y3-6|YQk@AXxml?JP~N-h8;qK}(#x6Z{!P4^$ykX9_1MLrv=gvH26#8* z56{)R3D*DN``vP0pU>||OLSx^`Y$463^zR2-cd^PklZ%Ei(V;8=vzK(VOwtl#b^#7i}*YD@_Galk^(KD+$CVX60lg{LFGx;U&@qQOZl#&)2Xii+kq$MJ`Qfv@-wdD|P6TH2 z$|UVkM9GW_(+AqwX-mq6h>98p!_DF@N6p4 AqEi_kS%SoYfYA^q;_)Iz{ED5 zs<|oV&U-^KkvWl7T?!>WAKAWCEsI4kBvr93hUQzf3?FbZ*;?NafT#ph=~>n>duM!h z2O{VF^u^|bX`S<$PNYB70Q!0458|H_0=a5wBvIO zkH{cu;r{TiGlL#AmHcX-AFQR{V`;;rZyK;zMlGEZE6 zAt)IcZ~43c-;LNnqRQy0DZ9rHb7oT!-_b1K=kM0!9%q2}QAl#Bxf+UV?s?+ja-hIz zU$Uq$ru79I!g`+QrV3&ToKQSN6^x7*$KJ*{eltGiO!@O{1krR@{~>1PwI>{CD8tWA zxUN5I;$%@{^0c|dGy5U53LJcrWda)F;$#Iie5^9%YLv??F$LS-S&FbY+|afd$$FMP zbQ~=ds`VdTvaA}ejNv#K`Gn^byE)*d*_~hhR%hXYWFw=0aQj0(B?Yb|I&dz z{0vQ&hC3$6j?zZ>=|O3A)yL-T&j-JO@HDD0pFC?dH)YCnztC9KvJfDc1W`g{!}mZ?5xYi9nJ~tGhfF z4zFn3T{9Eu2Uavn*(E^0@T(BI>h!X|0zbK6sI4#f?jxP9;9FVv|1xstyWv0uMq60aHtUBFak(h8E+ha zdB_tr8OD&T;rqWBOleN9kzDg)?~{J86{m1xsn*bQg%lMW>=z`#9%X&7eWHxbOoX ztD43=Hh>*7xG=*dryZA;+Fl-b8o!SXz!%5z17q)$ei?fyN@SRPgdPDlX~Hx1tZoas z$M%WZYMY^0c+;8-sg1?i!W4~%F0@~O|BS87ZXuF-Qhk zCEscl7z14A@ak(3$@2;l0^9=*vTPX*QqA7tm&50-LXa=`>o-Li`DjDA^2BW|0%9|VCX7Ka{Vs`S{KE4u zM(uyVPa$9fNuZ@RoN#+Sc~j!qo0`)PY*g@NdTU`M1MxrK$-7%x)Of=a%hr+@;yW<( z>IpfeR$;*4z&#otR63oBZ+QY77ag(T^o1(~>4y$zR>0Wm7mA!Gf1CVYN>J>n)_ zD{h;RdR_3&nAQD2%+b8~*=QjY70CRWMtQrK>#i3334NJb#$UigDUu1v*Gj`4a+WTa6(g?b^8@DQcHA({XIiC0zztf z*hxA4%dsNyt912*N~1JP6H!xbybuQy}a+vvkJzy)7ICiFg}P$`%hYThUtQ5uaTIo)HN;=he?(?S$I+7 zV_XX}MpkJI?vOL2LYbtJ3w*PH35dc9pT>`OW$YC_@)2Z!TsTZGg7Pejr^1abDe}f+ zKt7BZp2rB#LH*RgSc9>7c|{(KkmL*BatKgB*~LzNtR#oo#mwx>U~RqBW~vMMulsjs zYG_INcsh8xRJ^$HX>CtTq)THMA>qqdmh?A+H&Q9HoJ`FP3nEKJW5IoFsNTXrvX^EkFu^%Yl(4Rw^_o3Zg_1H8meHzRQ$J$l1X_ zVOo=WWPE|$w_+WI%1shnz^J^bR)~d$I91O)u*#_rv-t(~H+-)DFX<;o#%xxv8-*k+ z!%ca?{HQdNz<%TUugitlOK=JB2jXX3?!PL8S<6|rAfg6Ybwy~iW`KrgGywEoFHXf( z#U1^uZt(WXLJ=x!stdN{)AaPCL_+fX_#F!&gE)v;=}Qui+3Jz)@m2!;m9DfxG5Ig% zGl)^#f2hcPEZ(65Z9jSC7b@9hN^r?$0bqWwBiKy+l{@5DsfxUooo#szc&BO~{g5S)&a^N*GBWp4Es74X zPMdUfJtqGp65y+JV(vmTvWC(og4;KZ>>GkLlkhi2eI^y0-;8dCACjxD<78YUN{f)p zZ6~nrS<+C<$O%hmLsT2aC0UH;x~*-4Q+tJga8v~|k(ci}o@xy>5*uY#K~g zZ_#c(mHj*=jnrs|g~|fcNJm#mcjfYEBqHw7fd9mNlEfmizThA}dQ`Cy#H~i!rhLxg zn$6m2AwWJ$burkVt+}GMCQ2t&t_X!mX#-X1(VuR9ehrCZ@qxA;b#euKu&g0*W3}`sySW_|GWdQ|2wbfPIqoZq#t>ZhkY2LGiYQnf{x&Ct|8uL=JhBx;f;e)a*9h zmKa0}@+|d~*yS}DF7Qe{m~z3^->zM-4^ESys9`{@Ll^bJv&;$d8MKxXG>v>D_9Wir z+-fhr72aK^+t0g;Ykam<8S(DodLvy#k1No@TjxjDe&K17X9`!L`e3%-u1sKAvMi>; z{JqEo4|Q-Npd;9mzme2q*V|(3j_ir=(pYEwJtgIcF(0q_wi}GhOfRNM5G?T~mDynu zS$hPS3k$qx4<@I@s%kqs$>TKZn@bs)1v=Wy)|Kub1hn_LnInqN%uo%Fgk)!sYrMwcgGR9-J;+ht*rDCpP{4PbAw-~TyIRJN~rl$As zmfMJth(-~cA@;asQ)nH++R@^XVs0xfV#33a`9?+ymSn+cO*?>Mo~;u(j06!7;h#iY zBZSgC(Zm<`Q7sziJFA_fWWuyVJ#nxCkU$P{!x?G>3+TJbiBEIN&8V*36&0s6&SUdp za+N}6HKYpLJdKQ@Pm$;c6xR{&EhjMXkSQXPPU#t+*~+?0J0GDW-*7s_E2yRGc!iY# zP0{fcMhY2Rl>MDhttFo%koO?+!ICTT$vmBZUe(mYo+@jNovyzE&UK}VW{K~S^T6j* zW8w+LVFrif7()*CA%}Zq8FIKkNVbaU<|$6k@or8707nX7q%}5whprR6OS~ZcD1|>vh`%6fWkaylh&VTjd#wo#Gzu z7T;XGOer1^*KT1lYu@? z$Z{gut#JXg)j2A!cAHdGXB`z(=DZ-LNXv%0>SooQH;r^{mDdwyYO;t`&J*rh_~X1Y zH7qHdLuR8c>vN|SHX@z`77xvhj!dX&DDC=Ti6Cn4P*;|+AZtS;tarCz>M0~!v3UzD ze`X^Nj%{p4SdA}1Kn>=jv$oG`wMcj;wY-Q}y!EA2ryw2_E=p5lUl{=bl>8uNBYrQI#chUX%#V=l_LR&aEI@~*ag;BYK? zNaYe#Q*aZv8p%UG%_PSokWTBilo8KYX2Ys#op%Pe-W*owuBmEuQe8E0PUN!g<}h{? zv(CloIO~lTLb{MAr(;KZOUS|rs$Nxo`;-(B<{sMhW)X=zK7m_RX<-S>Bmj!ZO7nFZ zl*Ud0#E{ZC-=kyV1O6vy#OqP9+NOM`$9Q9aR{MptOE76$oJ;Q*m%8cOgc-cF=8d_y zudofOESHbr2`f4ltAY1u;3Ywmv;C^l{xy>`W_9(qT5?ZU>+0=uEvov%^B|tnBl2CE zniI(+b?@rLIA)-HL|_?YAPshk>H`YoB_7&#GTRH>X$|#aQ%}j{pq9$hRe&m_GZo!p z72sz?XnMY+@J7CCBuhtZ&A4N7Ex?sg&kav=se9@eh^^(Yd>?T0e?&q)o9k4^8_*>3 z=1+&S6ro#TG^vG=k~_l<0uob6L#AEKXG5^3`viee2^|C$Mx0B0xYdB;vV;Xv? zZoMCEVUUd>kj>$bt)Y)wKMpJvZb|9s^2b}=>t)b#TB+X9`i)#Wc%d|E)Z z&aUFAMfm`tx9qD2-9n7cg8+|?Ylxx9=?pOX}jEcCH0&ixQQ!^|$v;Soa3 zl!W|)X+K|It`80Zs`}?z=Nkf+`1G20Z00Us1%OT^FtDhA6boa{n}f`~TqdpL(F_Kk z+I4%-dn1K_jn~#^7_vK#wqx@pCA+VAc}uomQf>dO@PA+50!(&c$|oD!0=k7Y;GL_Q zZrbVPx~d{FN`+`(n^R1*RD7_aG9Cd=h#{vOMCh?iio$%TyIB}%n6X%L0m~3jr8ye&La?F_bPq0UF+)DfdG-0EDppn z1eZRu39S`G9j7eKhbMYES)?tk<4ocPt@4iK_PbP#k5>4QYjxa-)QV~ULmJT;8>QVG zWxie~Rk!O>00errz@^Lj;}%YUl^U7_YA>L?PDR~q|6f22;NYL53(PT{E_p6~2$-Pz z312?0<@r>@xeY844kWJf0%jYVj|dIoc{8K0cZYbnjA+O`M-u0O3oF~Wb%Cm z-{~$Ng<-_PA6L*65p@BBSgi9@%DeJwg_6{jY5LBA9HmTvE+`MoLN->Py0Ub-4t2V& zS?;r~3AP5~g3Z~R3&kP~dQQ1rY9vcjmjlyiHj!Le|G8tr7ECVn98@7vph^iUFfE7F z&&}vDhFKfwwXt&8IoKy<_@K-1?d*7)&GbeQArBDq5z=^c$CfqwuAw(|){~G{fnDN( zR^3R~Jldp9htW-g(Mf+1Q>~ed&dC2h{O>>fGCKOp;o;v;NB`p=e>nW_F?{&!=r2dV zxIg}IbaZ(5_itZ+^X~fcAN=T-zeh=UhkyNY_;+^skB*fb9sT9gnhvd{TIjZ8U%}es ztip!)L;3+EV;YJC9iTlC#))Rx!RuVnw5`2;*Xpf5(BDB}DCL(kU zwL$%n*3K0XA=lc5gwg`QfG&8a*iFGLU@%vC2hPm*sHV~MqyY_Plm+MlagUt93fInY z);}A_#Xfk9`L+um!R~6hYGRmdSC{naTT`iyocFzpg7(aBu%_7@)FJhBoGmvUwuuom zu{t>o8(DdC<|jd2%t|pdH~(yIfVttM)m5L#O&wtYcxbPe*#Y0pMGa4RpqWf9R`2om zYaTOj9yb@Eiy%?vneuOwC8A1IQuS&uJ`DmA+jlcnBo3!)$2VUiK-7N)&2N4RJXqw4 z76HRu+NNT7mpuugK1GK}tK7|=S)H!hJ6{*3{m!ayK{n310ykr=sUcQS|Kl%UuUitc zsWfs_SdALp7iiigj{`>jc*mB<7$$?EbAKjDin#!F018NK8WEEG(S7=J6@|53GqHtE zx8R2R&n4ni^Bj&Xvm>_<)>>fK$K(>ifI>ziWv`-!=JO9XH~@a{B7L*%0-=F9*o!y9 zah79p4VSF!8`!CD#3e?0)HTye=y$9J5TyHzean{arieqoW6KQvH>2Sk4qfxqnHx?0 zyj`J%@eW(HH_4QQoVDTd16Nx8r&9}s6K4{_CfyoBt8T_4AO37=zI8rbM>d_*w{Ad^ z#gZvhgqp4$D2sw7B7l;&251yUYO8bLU05kn7fulj`|`(x2IGR79lxsICit>hfgcmT zpaDa--+LbT9+fDf*#J3Z17#pFgb|Z!Yjyl4BL-!09jMgYO9Cp1*_AUPYXf51d3#Ec zjV6xnYxS;NVSdr{JT-obDcb%@+u{(-A(nD$TM<*ofYM!moEKG#D#ckOB=e7Q96a5h zWAfLfla}j5U_!E(Wyly|`)D_hU8QWxPlaR)Cdr{M;2pN$o<16r|HY(e3{_1nf_4+8 z**CKIs8ezX)#a^$N6rRmqs(gR-wrfWqoZzDB6;tORN17V-Ks>O_!&Zzt>TF(;10KKP6LZwKeh7cAI`%9-zZHX2bnM6t9!{c%dzl7$R<&m z{02Obyl4lnHjlUo&qJ1pXw$K;Qf1~%;LtR)d9iyp<5hDS`Ssv?x&8*rh7d*2G-jK^ zChN#aRdZ^@ArOJUSvB(0eNX`Y=-L7jR!gcx)J#Mz%V>B)JL|^RYMTMhekf## zy4^jXf{Sx1e25Z&2(jVN-<5h!rDw)C6k8`rmWNiVnZfN?Tb{FTk=f&pp>52%D(cH|ZZLnetPELw0W zB7jeYU{13Ga&9f39=<=n0mttn5cinteWr)Z_b^x8w%3Tbew#%=r#3qd)2}em2@L=k z#S0R{$<|!!CKrWKcCo1ZwZ>gwqbpw9?e3LDe@$fIfschVS#{zid9`r z!!b6y0T$J62oGqf*JU_$ituV*jR~`{57>PgE%V6OQGGAlFNgGAT?SNX?)^0o-rcPh z^{IBdZtAK!3v&>W^Y3pBC%((vot$koZ!xIz(I&ZE4UiTfq~6>aoB)Q-<+N;CLRWg4 zPn8yu&e(~iu}|o|VmR7~8Lj!bPSok*+cs728gn=FbH2jboTeyKX<%Z9LaI<)fu%@N z)CJfv&F)w4czi}KbGh-VnyC`uVA)XT1n@YGnl~($A7~f{Or4Q^_CbHXe@ym2OqKaB z(o;yc8K!1=9!;5)NB|RNU+4j=w@a!ZT^;j4Z{4KPAU&xgt{ZW#fkFiGep?JwOy9RLKWG6){rmVlv^hg9OuWK)iD(^ zZ48pRobH22Yg#o2Ou!trXNnpPn$kv8&2BhC-H?RZwFjgpb-4PRYFQ>0kq|PLyBmhR zV_Oe#9_6*3ji8v1v&EXRMV*{+^>SO2=;}lGHU4A5$kodBXUf5Ii7is88O4 z3I;xYFS%w7(a0T7FGJzYkiDe=xfk4E-sD1p0VNmq3~$Q%f@Knj39+=kjZu7ebJc4Q z-`!lfF_@2Hy3&2|Aj&~qYc3Pf0 z-h}8{-tdrzEsY4yx%%ABso}XKOajg0mIxBq9flT_IgcEnnF}W{H1d+6O3^qj&n55ECCQ>l*wp~wVT~?vw{5939G&J_;*ac@qzDdt~L@6zHUf` zxhfu*rz^z+_`yMF0^6#c9*a@&a1z3U3H$!XN^6h~Rx;g7#>ifACv+P8hd&k4K{sKy z5KAPtmgSvWmWOESkx#y5_4sogIT0|le&cg*xZxq+g|A7&zU$bf_WhkraQ!2$kL&Jn zkHzz8H^)2k=-0PJh3o~3=j7|#ROZiiu37u$#ZaieJy0azQ>J&tQI ze=-;MBP~XWVk6#!>)MKH+8K|s3kKwxD3EP9kY%9_3|(B23^D3E#ZqWcXQYrqG`Lfm zBB@ObYLQ{4Nf6-rY021eUfp)GBN1qg)21*HPg2a;4XO7z;F_cv(sLoQ8jDMDTlgN; zsR&_u$CgAwkhPGIHm68xOnWd3Cy8QmjMYDp*=;*qp|3Zr0T*qZxxHc2s;RYKr^aO` z7Scq|S){oIfUrVHrGuBhdy+axX=_zjju7s+PsV+>_Y-iOmf&GMyfw5T{N6y5-~(_^+kqUMGAD z6m;o`4{6-(gg3lwTU-Hhe{p5AQrh!hIxZ`zFBCpwCa199sUOy+KF9Jzcxn5sphd)N zm-u<)*Dp!e<-l5smjU~owg08g2-IEQnQv|g*8QO9ebCFppu8^OQvDc=u=P>wXA85z zePWlc1W!Ye`^XLgn=UGObb0(qYJVPhuDeOy4<#68=YUO3rv;%-?n4&n)K6yG(gfWW zzcB+DZe9XG-o`At)zrIdn(9HA8-%&vxG?u%9v*H#a>$5my^u0LPh?F`^-OtqM-7CX zc}?SPidy${Njyuxw%HQF`8yVI5>Q>gUl8>e2C#uVWf@v8uZ$Mr$yHfA3_|nGgzJpW zE2c@zq)mbjFzB9*a{yt=2u`-TjzU;Ogr}*6EFcr6?-`58>FMby$h)Usef1RqY$Ts~ zJY0Pcp8DchEl@6&|2lvD3fz}ttuXsAwrXB3RIHB4|Ga(sCWFo_{t8NvbXuC*%xe-) zzx@33mZxpnQV0>tLJ(r-P7k@Z;kd}5gIYPJ#XT0H1^I05(B}>P2Hexg_A$^C7u`Cm zQws}XMeG*+zR!+H!0#BDs&Ax7;^Q=i)FKvI-*;>%qp4cJXDr+GOYE1NtYyPMWhFWN zm@w+i$sJpcEVel1sb4(!bm6IAlo-5*C_e&OaUH91kyf!3|3V^21IFDw(Sji{az7W! zf?VQFsj-bahDlnDfXAW~Bh!L`ts?mZT$$$Y*fIq-Xso}Y#~Hq_p>Ks0lSC^uV@O&{ z?ZB6zqts!jWc6d%j*Zc>&$?AfC%4(ikq;+lIA=Bwe1ObANDR@#-D|#Y+C?4AR|U#{O=CkwcuXVijR3Tg zEHaAP*=XiEU4{5Yz@C~;-eveawuhF++J(-}x$Ka$_3T)OYXjC>)W&x@XKlz~&>?5w z&W~3tk8#LiS%qrLZQ8te)11tHwLI+EiRHmgP%K@~Q7qj~Q!H&ZXDt0Uh_goK zOCn5jxw>m~4uaf2)p3^PKDAz&AJi419hkj~N~kw{doNYcZqz_KQ30)^{&@`5Pp9pY z-hI1LhI=86`f~6d1p(6>vAi+s8bPVoff-mt-VN2p9d=`X^_8c1H|2@qP_F>D@4FCPi;)76)JS@Seb7vrwYa#wBEIAcCKWmSD{FA; zwz(~8BbqvCWT2(3DPy3vs^s|DsBjW+22@_J)V%91LPD~56}O@~ds2&? zDpK5LSVK{ko6C@Zm2frZw}yZ-GGG>e=Hj-_Z6C{vRmgjgp1fU7?-s}84Mjy~LOj=6@=2nbcYGgY`g|W+m%6{r zGwU#jh)mgoU}Wdt%Q3Mm$6E0iZ#yHf#&Ie@f$W4|9w_1qFG+K0z>;QuM5xeeUz|qb z?hwG%BzUy8;EVgnu(tE-+K}R6IC1uTUWRvH6Z1Wth(x;>vj&sss{Qoo+DWSvwpx%x zu|_*+ri|@f`(Sn*%&yhtH)(dgLc6LtF#`*tR$wX)kFR-h`kK66X6y_l{c`MW?&8rV z&g3!4o;~CV|a-FLf^FbDhhZ%LycAPU%$D&1Fj0J=mNE3w#9FA>5 zonz}dp5@I26EANbHx@{U?vBt#%nvNxXA5eN`Fkc+3xZWvidcI1rkf0xi<%s#+yyO}=-2vXC|tJv{Uf@Qs{36_U4L$7OoK2DX?MIoX?lY{wrFh4h! z2R1+Fj=gJZWL~v0cbK!hTHYC>C(Oy@R^oSdW@}{ba+u1Z&(Fble1Kj**(D;;)6str z7J0(&N<*fPQfUy3GCf(|t{%Gnf<3xJrRaME{cAs;dvI$9KKJ!{VofK;mkB&6)SaD_ z6wmey<(e$3C*})Ild3b-V$8#Gh_8*kI61aej6)U*xjas*F+UdSwND_pvzJK8BFOQX z3Pf9$htff}VR~exxTi83AE<%T^A;0kVW?lThh0`ubE#k0mZJKV$6zW zVJsw-OX6FN*V1WUD-nbq45=Kt=kq$o*a@PD`8)}?22cdG_DZ@VzZrq|LE%3!NH*SRf@(XGyC1}zkKbPLwQdk;hC~6E(KK~@AUqnR1 zG~{JkL@5(gsDVQF5$-_-5xCR>4RoTZcaYBb1Ejl8l<6U4n$5kRb@1LLKhe1y`Gt#o zGjEIa=aTL628K$H z3ZcXbZ^#j2K1NvuS#M<$HS~3I>k19e=sSI|gbso4ttgE_Nc# zuNqlwMyHj{8R%gXH+D4gtGn8iV9gL@_4tGY-!_*shDE-}JUwdbJHNhCd2IylL7Vsh zeh5S~%L5svJGiz^bM{1#>TbdPga&x(G!E(F_6i&0v$-DKGtldz7?5!&*G0bK^ve~T zPmNjbtZo?xcu$uAerg>|3vMb7!XB%hiuyw*bcdQHjD|s;yL9Lv&4LQv&GgIHuF^^S#cK=sy~M840d_;dDP!#il?k7!`%_xk;XjatNA z-`NT5o#X3Knvl+jij^B4(D^3J2L|nukhiFU#C^o1BaZ9$7srH;+1UM+R^!eZEoK<) z;3Alst=hsL;}G)t^L)Mk{G%D1`Ss7EKjxD1{P~~eSZncs{AcNnybFDr2>GAa!oQ7~ zH^!+_|7^YO6T%`bmt)M@O#|-Z4uU>Q_Ah5{fn`bc$R5=!f}^}91zlEba57X4{DH!p z9D)iIfj5qc76O=56WcW9JRI(c^qT*rt&IvD1YB+|S- zPK4MSMcO`0LrxBYi1x%Y$9XK-$VpuIREws-CZ2LLkj>q3O3HJJ*ZJ6oYOOzShfTNQ zE^o#zA}^DxU}4pmyaACZXeL^8R6=cqwM0eCV@grEM8{^XolAP6*fa@1a%`Re8evLD z$vhD$EfWt`ijs*GcPz@{Lt2Y@CZBP2U49vnSx^!2{uVZ+DMVd>N!`o=_7GZY8ZYrE(yx^&%Z8VUIO)c(*NNAG0N9B7H6VZZ^*TM&d#fS%?irYAMjv;RE0g zp$Z@&$aLqlKI3h%tf@B2WaLqrI}N%tf-a4ob!fbUucRwIO_beCDXy#;>g|f*r?;IA za_dk-)`C5`tWQkpF=fMl88_Up+l+ozkk+j1p6WUd4Wr!GrxRuQ{T1xcz9 z9M`#XfU3`VYgV<{ZHs>IlxpYE5+~|qjv!E-MP{Zu=GNtM)#r#N?MvzS)gt%Q4V!kG z0?_y6U+Y#sp5KKR$C-m}Igx$0nXyz*sHW%$S86ej>{QvyF75VnaRH}aaBb`M^m;6U zpr+~nXYXH^hm{GOzyFTu z3s}|w3)^af+wG6h5O!En`L}@gWSoOGgIm8f=hs&_v+!}hhO50Y!k(J*Jf37US1Yve zoK|M#l34@09~}MC2T|v@%2C3WScjbP@K#SHct-*&xRq`2DnqhrZL6%($6B(cmM6N( z@aPn%P{TWHhW;0OjV}IV1H)Lh3*#nLF1UX=MYCF24Gcq(0+a(|JrsW0os(d}XrX&L zC48zMSdmV^IT;`8*ISPu)h-wC2#Q@+=Ug0 zBjs!FTPZvyYefy7#+r-6a|ZScSws8s*=vl^GfSwkOHGS3!jMg_Rz5IpMOX$ygaor&n20w;?}2Aj za6|&H+zQ{dxy3sN#nn`-)K8Vm)(@m1FQfkyr6gaVNss)v1fFgpvia*d_BLoulsj=E zoMaBDQHJp|V!|$Hp~QkJ9)Y!vPd%sE!lCxK*b02pL7FgoJm8u%3w|4JlD98r*Q;A# zm*>ef1Lk{c3n>sa&lw|pq16!zz~@s}8N+%xyUr+7zxx~b#l4=;&Y2|QK@#;Kob9Oe zlbJFKr_})iA79h6X0iG#J>Gb#@81JU68-;O95N{3rEzV1mSX-rm(~Jk5^9d)Tx6NC z=`T(;TN?y!eW}dpA{SD= zvp?DK^hRBB)Fnq<(rcfpE=fwRvhTb3&X;_xUkv%FE1z?Z3v9e}?dN#m{7pF^rGsZ_H+BR3}8 zs5ba|k(`(*%KHkc`OQ8ew=ECXULOhaS(`RozSq9Y5@jZ(7lG@Cig}k>bLgXIqoK#!_*`kCZxGiQ@T$RS(xQ)$weV; zaD1WgNtMcId~-f?c*-`?6@o4D*xlBk(WRoQkkyJF5qO{v?2&Zmx(7iFvj9R_{qUM) zm0yQvfRYWfK*?sApya8h1ytJ(k}k;>5B3|(7{Q<488%}RT*{=JNG!5KF^?S?rV##2 z4ZJ&*f`HE#G^H~(al=o_+?MBqUa4T4HgSpP<3$=TQh$4IsHJ|D%sbw7m4>Qv<~=v# zaY@92E5&S(th*YggsV!Py1DFNhsStV_R)c^mcn!JG(b^(;1b2+l<-A{cn@%CCnh({ z+MnHwBZ$2t>?3#ua8jQw9~ty@Om+{ezEB}l`$JPCj^43^{8X@fMYPaoa|OZ#GypA? zJjVP7>(AMT#RwcZmmjs@3-y#X(oXB0RO{ryRs_?Q+eTlk#SlRMFF{3zZ99MWI{a!6 zh&v-&Fe^gN4Usgh7j|L+o#27usR3nt$Q1%Eln&f|g(x$nl| z#K9E=!KYZm0g_40{@Rc=cSc;2tCWb`Y}p2+ZHC1Gt(Cm=rZ$ae&PC*vW7*l@W?-h;o@*5h2T(&L4k!>4d0mRl1 z!d3mYrKsSa4Au@e*~A@eA4ofIAdTfPdg%rqX9=r(W~}yJHv@(Sr<$hAB~2Knv(ePy*NN$xoOx>CWcADM!l9nsv(>ZH0hPr_ zh50?ZN}rWgCMyig31Hwse+HjDYZgt1iJbe?XSNLuGzQd*grWyyUuO9FI8889H{N$j zPmxJ=d=wipR9gs2XBl*f>-Z6S-~HABy*x};epm1Iho&ERsi#~HIgoWEXRo+`pHU%l zlfJqZ6AGADCE!LsN?qB?n1P(hOknI};{uJ;G6y_s8=0j@R^*wG2H>bp$ibN-CT+*& z(P4(DqilU2ToKpvS~)Fr+IS^`QqjDCpt;x}KU9WXg&3R$rsp1EOor!{(yD~=BX^Xj z&Jps3v1UM3EX2vSpOE-w3@jn^5hBHpJ&867i{CpGk!d zU0)?&2AtKo!DCuA-%1h-hPw|Vo#LK1Z%GISx@{(&k&-FW`IQ_O%jN$h?d+n^~7*}AO?T1`q$F##6N0&R~$VQS61vE9i}j>UB9;iPmT zCfT;Zd(sCuW83Br2vicc1*9sq3d_j*E#B=9z~;o1d|e_DatStXZp%yQi^0<u8 zX>TBel{NMt2$J{qSCA(GV$D?ai4_>@XE6^E7)w71AIw`-q}{?s7&sxG&ENj^E*wni zsQgXaQ9p=0i63~}y+R0nB*(g=lP;17Fl`P3yxbE!{=);0!*{WV(4e#G-dcT8?=8`r zOL)vur7w9d$uaz*yTHID$C_f$H(+8c?s!_(tt-z}b9a4Tt<_sHg|&?!y2~G9*hdzb>J3>^ z%y?_rpFJS5w?nR%n9_x0omCQvExFZS#qaSD(mzc>;Dv)Nh}HbsPb-j(MT*7nq9j7ri7-ePISE>_2V$2OZ@4IE^^n1 z`_hNmFSn@j=L2HzBfB;iPgZjMJ3DW{!7npZO0~e6?Tg zFO9hQBX0g@jhnx3Nb)JvdJV(EG2gA(l*}KElcx&UTI-+zDgI$RYfe`BR(`0wg=AVqX85By~Or^45Mp=P`z z9Mji!tL(i;>(^Y~Z>turby&T3-&|Z0^Vh0gST)zSc0bIF+YR-yB^=8vVx=Q8 z=JNgt+J)oOd>Vo z#?}T5vhEUkVmsY!Tfz52&PW=399MEK=vr^d^U-YO4UDTe19m z|1;>2;w^N8$Pw+$K&+pg9i74BdvtbW#jeB!aGH0m9558;r%HWignVaHT%qJ=El$*h zfR5MN;CBdx?_vLhgPuX9AQTg7#*?t~)?FP)k+*31$ zNGukJTrzhxzud9(JQsbeBWMl%%(dLOjTtQB#Y|goZ7_?q9o4i^_h*xqo|dZ0&d`qy zMw7HE>ur0+!NZBgNmqYTZJk-TuBR& zRiZ#;EMIU;J=(YIy0_imR%~^5vt6)lN4)Rlx|;o6cU+6mgSY)ud2W`oiNhF~@&xip z%17U0!L1!ztDiL;;;)lc-TGIGg=nJZ($s?O0!MS$b;h7YXNVD!n|hR4&X+u4GeFVg z=}eZTyo^JtwQES-VdWF!90+g0y;QXbeI|Oo?1A5}w|lI#igcFl7rF(H~e+$r(*HW6?X@Y!b?z)eeotV6fHn{#Scl z(L4~$TYLP@Ph!ggECE|EOunGAt>fvSlM1U$$#)l54(Q3(&+)q=Cp1@lN+WebvI2_# zL+Nw#aGG1w;Yu#c|dqV+Tlf9b06Y3hjSDiD?wxws9XgO+S zN6<$Xb9ukjG+jcVzs(`U1&D+>7NG(PGvjC~gU>9AupS_1Chzjp5SyQ8$sPc`Rh5{D z#?BFm1tWAxc|z|Ja76?DJhO{z-{Nyv2{^fgy#xAW9S!1Hn?f1>#lWEx5#9SQIs(;O z=vug?bVXOz(b~rBvi39TAdM7k5qd2m@4NkAhrUE4v(^21Z05J-XsokIh(+pNsmEu5^6cUO;L=oBD5a5ONMtJy_lJM|``OYrsl(6p zQ+fi(#VnuMvi1v_@+p)0q*t0OjG+v$D;_jKAP#+P2_1avwq}0wPNr0uj&pHh&~wuX z`^cg~!F>R?Ejdnw{#%FZ>ZZ6ukqWx5P0_-yDDNbCcSsg@y2Fk!Z|Ao3KlCM8RODQ( zG^1i4)!9@+1^5fd(>9OJoJx`k6LN-xUlOJ!mTj!VnS0!JC3u-xPYqYmBvR6$JR{da zO0JK|)xYS81$+X}sBWLoLrOqpc-l{C*W7(E98X`5@Cs_lDWd4CRt;X{mF1R&D|cX5 zH!IrJ?XVczugF!M7^>0BrY=VJMt#NRbjcd|Z6@Ng26hD`*xjj?cU`EktJfeUTR=j2 z3vX|FqV85C5dxBhi;ScqW}0}w*-aC9N^Z{07c*~2ERX}DN5-OtFYHv6^$LXIo75d8 zR0#qWCJ4Pm4KRhw6SidFGMe%k!J4ECQ*_A2kBA&fcjkRrc=ab!Z~ueH?-M~|uReq% z3BPCLHOmsQGL?eTnA|9;*tAG)dbVvBA$MFxl5edo>P~9f;?>Q?&DB$mu4}WW!eVyf z1&QV~J1^AyHJ4E4Km4&?`=wVxfq~)A!V`17&E0I-^>Ojn^J_v2y?M$Yz;9_BF4uU2 z1-J&U(EM(T_1PZwvARPhwvEw{?QKi59bD;cg$kZ>#iz=dkxNO9hS9{z;y*%ag{2TPie$z8ONt!y})s~{Z8kIe=<=dfK8bE`PHAikjg)o z@ZM-G@gN%;|2bI%Q>XQtY^N-BTkI1o#vd)0ho!7-kiqw}ma1FP3uy0&ljIJr| zI4l+Jz|KM*On2`1{N-K4XXFi+3R8~p%aG%S^$4^CcCF=Ap8ZJUIDdt&sWocm-C2oR zOjZuu$4bA)YLyw3*4v9~=tYu$`|<^dWWWF2moGZc|C1F>J^VZ)Nd|Ixb{?oD( zpF97ljmu25sL}eFzmc0-VY+xz=%M;)D}8lc%JL|$-`)K9=KQP6H?Y$t14iE!p6)zn zaqBzTd(P?h!MSNIEH@?ZzP=e4k-Mu6tLw4pv~6b|B{IsxtE!k2^o9ZYd8hEb$K4=L zGO|=}3vShui7DOncK3^y`H)&yy|ra>uN0QPmB<5g{XXfJiAiZW;-^BBTg8S=%n~N7E0h zoZ=e3apAnF0UQX^w4M907N2Syj#Yo-7I%0oi}+~um9uz29Uc8;yRTQTjaX|7;YdSg zB!JM!IZS)!;hIW$Z zQxU0;(ru&9+T~~n=UULrM5x8$A$_2EOwO;{!b*b5Gaw=M;in8L&`@>>LCgkKhmkw3 zWCd-@GZ1LL>AdR@KwW~y1d0MF5s{*avE$z{^}tw4Uc7jL#@82r{No=X-jT#C;*0e! z!ngkL@@4mM|9<}V4WSXp_E_v3M};%-JC>*9|9|t{yAnUA`YDit+m;?#*#yaRNL?sR znpDkQm^cNWikCWI7(h>?4U~=LvpH-bPl3!59+^TIXyCTRi!mb;CR-YJF# zn#l4JelTZJi~j~03u57{40qoC8MK>NM}Czl(K-AG|&r$b@`D(^08hf zn!cn-U64NTKzBFc;N1}>pWo(%R9_W(hx>t#hJ#66tc`;HN@IBkOHmXtiInpXjG$KK zxBr1V5HOOMHVtM|j(7@4&4m&Zz#Ozm;~gnxWjjx@IXwliFDKrukLA&c4-XbJLdVQC zzcB8*I}2`((@5!pK`cH_DPWiy!iD6sl;vnnf>fy*h<(u1zqMHoEZ0m*5;~g!rt6ZK z{4)N}5>sQ&$o1u0n_nl17XK9MF6e{Jm1@#tQ%c`wgj`CY04hTFY_?uH*g6QFnm{65 z*bbG^+?ahgXEe6yNm>-kG8Uf^a-px;lt+NKkLhZ0p@ns*i4`!^?VJ&j#|!}#r{Gt2 z9eR%*)HVQk?K`e=ny>IlG5a1ckY-p2L<#-Dm=Fw4x@*wb7S2<20}{G16cmPwycxhT zjse2u`t@;1hANtyXQoQpgrK%TLqUyAIvBUt#zko|b*5!9RnR0^P1r}xST*s(g8;>* zEcPV}#$5-Z&2mgzmE;+&BNb37;u&@bK$eBVSdP=?pM9rbRoI)jlog1aO&YN$XyuY*7j=yxz&^afD%o8o~J$VYBpxv6wvKfzji7UaN6 z_>|l;!-2H^VZpZbmj_KGXOLZniE?ZO9pV^6fc}TmA+~$gY2j#Um0i~}w$d)_r~bp> zJs`GPo!xQ->6Ju@A)7!pgO$f-PjIX47)XTvxK%^sJzE_~qz!$$=R8ByMa}E9MlqN` zuzi3$Z#yJD*H`H&c_)gX zhyR(;v%GEw4W2aP;V6(XEvV|lMVB1a&wUpIh%DG$FsxU?! zT-O!@2Lqkb(?0>2)K|E}PHx}@wR!jZ+(!NA=LI_@u}F`UF}?Wpkm@Bb;)b0*?m>Jl z8_gM1->@~EjE;^~k-)A7iRw1ms%fy0+zd0A$Qs5flYRu?!@7ij=* zEOSNBr0`FcwRxmM+2Js~rhXA=&61$t~>w{uq7d&SfMBcd65bJBAe%0-F^!03(~W`Z4fWZ^Pt zXRL31^q|KOz2R?ZHqcAsepkWPq<5lFHlZ6o57;iiV^hMqzIJRl9-!uIb`vllp&BrA zN*ImiS`?e0b6g9GRw8_@*NYOGsJ}j;lPnipA|oHBWyo*fXx5Ns z%NH6G6|4;O>KfpJ)+$)6f9CE};7ARtD9aUS+BZQPMY=WyV5WzUgQd6h1PwCCa8S8ZZ)qmc+@a+YFm4I}j!|>tnhIAYgF?*9wuwo1 zpgFhAlxx;$XnTm6W+RPv(G!ByIqiW4KL|9b;5Y{bC>a3oPyxuc14k&;+has5)!Wq+ zYgslT-1q@wPLAmh=;M0i6>Y_c7x4Il zZkBlrY2e*6PYGT>dM8Cvs1VqYOub`tWZm;NoQa)@ZQIFYVoorzZKq?~#>BQJwr$(# zcw*al`u_btyzht8ySnx{wf5<~s#oEv+Rd@=+;CLZvf8^Sid`~SoDgH*a?|ra0KLv( zTFGG=Nf4&KHu&pP{fwj9+Hqm4Qz9~Ut5?bBVJZTuA^Q9NIS$NlZ4&u8x$SuPg~eH! z-D0;=R$BLH!*VXLk8=y5+)QZYxoAHF4}GMy616qUg|JOJ+^MxdWan_BH!b3sCe&ux zU{;L~jqcf+tLs)&L9dIodK66WE&xFkYm>~Hm30mM&rqx&p-uVFV3t+0D1;yQS%=Ru zd>s&g39~vzIStl(D3u8T~DH3GR;qGZ} z0`>`7_lz#S+Xl_G?uQ1_c#E_@6%Bkuv0CS}O&79zw$u;_N2g^Jmq>r40i2()B#;@= z>(RS+sNl8k_^Dth1-?5*6K08PI#~VW)vgU0RWGd;{V+6zyo*8qu3bnrqgkGiOx;T) zkw=W|L-{I`EhC9-unqWW1BWF&H89o_voKGkyZeE8k%43gdqilf=+O4eVPpaYY za=obvZAxj)=oimt-1a9Q&&$Nk4GJ^W*T1VC{?9n}=Quz9mx-~NN8%oS9v*?uU0}=h zhMxXscJBMTs20QJjlLl9S1;2S(dyXPn7%K&eF5f4dZ`uJEgaDaeK;z?V)o|c#ih$j zPHOm%dzKY&UkwF{Z19+4#^7CjoNoDgPJ4s^kDH-FL+())j9YP+rRBwir7RMl$!jg^Fxen1Pw-%_>ZXRrotFH_yt zrmyIj1?rtZ?`ujEj1xO=C<<)GFawhyOQqZiRBpUeLPNOu1_nvq^HXo1Kc8&hUNV(d z%5TDLhmdF|`7*8<9{^jvU1km*d>9G5fc;o_Ruox;)b!_qbe@@M%jXYY6}e~sEwst% zx#%#k$yQtxW07;?VW+%(e5?qLt%@!yK&?_BWEehHgTn((D#vb}Z>7yFre3Z>{h;>Y zU%3=&af_6LKnW+RE}ncXv3Ahjk9ev}23e(rJ=chRTEjz5h{ zxi^#oZ9g%r`e^B=AO)(FP2J7sMTlAqd8cn*iUiyP&Vxn-*26>8oWS5!H;I$t{_SK~ zf+w6G0)H0=Q;-{m6i#wvr?)T`AESzeZBMDM&&!ZM5WU(Rpn*j4Jp62h_1?136rGOt zZJ(_#KK(T#)k87Vk(VXFSMB8_K!1b$-R=s}mu_%VI>gX4?Mlsh;e+{Zgj1+*x!4Ab z1Bc9O(oxYl& zndB>b<=P%Pm&{XfGU_NYMLwBvBu_g(1EN)1)hY~xvLlSVaHJsrh8<>J*MGGt!ufvT zI_9`jWxN5PC}d;lUzB!dnqXd1{1p^CxkSfeJ&vhnigFF<1v&Lo5*q#EnD8)+Z)-T| zH@s=7}}`xu~uGFN{1u;i=zP^P8coCo0TG$ z$ZSeaO-Q*3$Bm;eY({D#i$1m;ph^SCg>|}%2v0YN9JnLOVaKe9L;lTXUIZegLQG{G z->bZdN5>}LeC5t7pX$2~md^}%Be zZ+;L1X>E2hcOB3FUue|rrOa?qwYSV6>c5yd#u*{>&dTQPIcD2$)A-D7ZlpqwSvmVg z`y+iqumxM|jkWjcsC<|@5>Rw78x`F7uoBwd_*(vhvX83^vL`1hECMZ2q#o{Sh1k>e zUrrT?TCI~{G|C>2^4wjx>D|$xI`6vJg0^^#3V3DDwHxD~&acylr}~-0eYiDirw78>$*B3PEi+K%+Q zFMHdsAf9(_mj}U=M8Da@RcpZ3WZkA+1B;2wKk?EKlUj$uTbh1-X#Pl+TmW~ZqQzVw)H-~i> z6hGSD$NUxefYugFD07uM7(6Dpj4NPONbGY&WQ+DUow%EHf`dcK|CVmz_CJZ|5 znxVkkX`$So6wU`>u8Q5>nDuGaD=4glpg;#^9g`y#y0M-|yU?+9!ec2!aQ80Y(Bo7o=9Ul@S3CfxknEmeTQTavUosX6ueKkun>9K<$)T%Qi{iL zj`*zm-6f+;|0x64=atsam=fv@Y$Gk(f97uz*Z`&cH_clO!dQ~b#u1nXFj$0ezleT_s^-;;eZp7)Iw^ITJ$;l+Tr%==aS(Dbz;~=f@zB(QNn7d= z<|PepAy}Mu*bMtdnVz?a4U@tfpH~q`_;33foJ92|VC_no3m?$Csx}%4eF|#?MD9xA zh0#dd&`{D4)m#s5jmr`gj1K0=$RN_wuu^2wG9+3)L#-L-hcixyo>$TgGD!^*xVRRt={ts@_?7LHq z@L)DP8yxwk?;AKWofc&h75kPOH|JH`TSIbRj8mt}ZQMD>m#O=J8!p;c>*tdDeNDC7 zu^9i!9lgnK5`jqGpq-#|pzgdtZ3&IJf?NRAZKYIoBN0A%nU+Ta-4K?hywBBK*Uxr! zL8qrv(k!MsBjJUJ!+;&XDzVK)qJQin;4G5{PJNy5ZFAz|esmg^7xsZO$hu0i+m|h{ z2Mu_3(Sut3nVOkEris(n7C7j?X)B9a`DyOD=;-wA&AZ?2J>D+jyMGtOsn_%+F}4|- z^xf&~53?RJL;NdkY@T*pG-Q1(ew*H6G5rhq_cxq~@+8mF7oZSm7?JX}UB{#AsCZAo zySbBjKlzSm5}QnuMcMPmQ_*&0|FzB8^{e`wG>yo}P&NzwFb!W8 zqvc~ZT~TL>>d{XTvoV$XdalH%xhyTn^36oJkVn3SR;!L#(F>zt0GtJsF{#!=LFw<^ zUTYC+2SfaDyTwfAuTE58 z2Mt|>`BRc~yrt^t2&^TzU(!u9Wk%MtnsfZSjF;EV-cS^C&olVv*H9Gp!`IiqmEY4) zPv`r<1h)R?Zk&Gi^Pbz+?VcNm7?|)QW&ZiPp-Ep<@Y2({>RI`^ClJq-{@eH9+cjE- z!=qvyhPCQiQFUrh`;p~)qz;Bs#E_S$dJQM{?)RV$gUDeOMOjNC4puZh3Z>*I3d0wZ z;dypOye&+=2u{XSSshQXucA%QI4V`9cnT%p3*=^3*w zylr+Vc=pHPu}a%CC-DUHy=ZECz^8H~Mvj}KnfxhY5*pLIcDI4jTu4=SXr|w#((#{{ z!N-`t&Y_40HZf9a?XN=UL|^4G>Rb=02uEW>Ok1gb8vnAMV-938Oq^cPp3rM^8CTG8 zWoBw>1l~VOZjY+N7KrR{g%Azqne_jD{Hc2}n>pmJN4jsyd>6YIGK~M%fF9s&Mlnx0fjOp0VMhB^dIqFirBGol+gY1b#{UggeXlix z&|aVH9<_Le0%(8m;7GeO)6tYOA^Tc7UI0SSoGg7!Xkdd54-Hu z$NX2$^%m)>n$Mez{uZu@3<>qGPL;zfLE6u(`!>Qd;|CwRxxr;uKb5X`u5!J?(+A6P zy__>Lg%`Z@f;?sSZafiK`a2$Oee2PQeo0*)o_}O>7+4W1G}VrkwM7YK zhu|N3xNv+1JLA1QO7zU3bPk&D@`=#$J%m)sfeKbMZ8)Rj%^A^Lewgz&MT(J>IoEqT zoc*%9s^zh=8x1))<+(K}bt*#&EY18hU)0DD0lsRR@1Yq-1!=q8X(|b938;m1mUuxJ z%ecTNMDCmP3LX^(hC{UT2+I~D`k7*MsrA=yIN6qRZzsEs~lgroxDUU2CflS~+ z5f8hhr~qao@u}Q`;|!qHT*{Y-hbD!2g9~iPAdiwMIMV`4Gj{*FgM^7j?uQ0xzRoI&}#d(6kc#}s=h%$leG-0_}D7j9_k&}uU zx7U6Xu?XA2Tt4A~)U0S$snR}~P}8lS;Rb)D@K+`^a;1tZTy{UQfqVk=O15%9aeC z2IF{_$o3Q@azvUi1SNr`HCV_^vNQI+b1~r|;%H7M3BO5KnH$4`$|ZTDqs>x-C191| z`wIm$%2%U*ne^S_4$?|M^q#MIJdJP ztOO1Su{HR1MebI(6*lfT8tZLovoxo#)APuhM|Wm*dfyaWEhEz2V`v@H28&VWB-h

=(_F3}K+lZ>3$*-($7Di#@%Lo1)`BYEGLzUk>WIc;X%oCGNw)A0(azN&5P zkuM&*p{l9Eqw?KECZ!y_w|Fff6W-QS*3fuCoz zc6@}`q}rzm`U!;k6&H%^@sXwc!mhy zZzdLTJ1WPP9n~)BIw7K!d>bleDL##YOsmHFu%rYx`f~Kyhe9`x@-yjOxd?Fqf#VAt z_yat^giIiXn$cm;xO)=RrVKEdYDrwctZybqsYD)$adE;E~t}YC(s|^SZ%z*#!P&g8;Ldbkxmh@3R%g&=5&d4{Q4l=!z@KO!BV}~5YC6e z+>KOXrERx;Lq~j$p!!_CWW#aeUAZ+chGx&B5pG^B%JTX8~(#w zrm9=(3XuOLV9G!V80d||FK89JGE8V=&qqOaXo8x|Z=&Y@ISNC0=`d~>wUM4AVCJ`g<1}ioA)+f*W$4!wd!p8sQ)!g5B7KQ0NdIC{#ji9Y70%-& zDdW=nqymG2S?|ATv7Cy9zmYlZOqPtR%o^~;4E94`8};Uu$cWZKlM{a-i}p{L_Mdzy zDmr}$|E2f>i{PdlxgJRL3&#E2bj`<=#jwx)MmF|acG<1oIyDfw>0##`>4yly6{{?! zpo42DJNOacd)7R82z!I~pzHTT4Yv-R_MX|as9)N`%#dmjfszW_Ez5L(#@MpaD|dnh zjDpgOq;HQMo`(L3D${H>y?0`f3!eJ%6^bQS!vk5xOVx0xo(MsI>LOgu`*b^uBR!i% z!(^wGJYmcxPocV``Msdh*~%lc;&D}@2?pe6cWhic8`7cx5JfcsSG$JfZmoRZM=DV` z*rwD<(&0I8rjfd?XSfi?Q9RF;G5j!CT(8dZ z$c!UP>0!3{B0zH5(A!~sw$i2;_jT4IPse#SF8s@H^cV&AgS1f4)DZLeeL>>C`Im!dQRxR`{r0jN#EffQvH85nU#5<^A6TC&yO^r_eH%Km4@;Yz z7k>UJ>*a|*t6Zv{s+N{!29FDVcg5KaEXK)7zAi&Q<11wIsvSZO?yqWb%HOk=h^Pww z#>`FHws$BoaBWg8?FU7QB#MV-_VHV3UJ^Vxm`y!Qb2f}nMVM6eipo+Z#jL>hgAV-> z@np*mh?oGw2{0*_MpNtfFULnOEg!uP@vCE*)ATYcpNpxF>T-QtkCP zqy|NC{D|$*MNI1>1dDU7pnEX!RU{Bv5m7{=N|y>d-ayqo<*r)xNgo7t5MuM~f= zp(5Y;b1Gm%>CIJ&|KyYd##NYq?I-7(J%3EG!6|?N3Y{u6xFdnfjU1^cQ&wY=ye*b! z3~-;uRQL_U=Z}@>0C;cM{4%MMDKgg0$0{uw49cB=clc9xe+NniTEmlbS}EEjA;BsZk8$2eIJ-zpAJcD9x>^O(Qhq#{z1y>t_FTvF5~07 z^Qx{H<&LWy$=M};qq#hR68JuT_<;KlK;v8-*Xx`F@iK_h74sh{xPI*SjqCNABR1dJ z<@i2EwAl8X4e(DOVSq5zdcT+2rI z)^$jD?v+DWubFjN|Hob48rHHg#lKOK=uB#UoESvJxls~uYV=2CK7eECMrAJE$^pLu zP5Yt}*bmC!{_nmyiR~o`{11ylV!(Z2jPGo3VT`~%H;DS*Ha>E#A5U=oZxK@$Ww=-% z!5}f~pkZ^j*x4JlS#@!DxrIcf@?_&|+=rhzjD1}$UOE{&p3V^qf{`rS>O`0>9^Vag z9QPtr5gsIBdp*})2c{u0=7e&_cS^CqX|&DEx`*7F zqA(i{?li=1c}O^GIgDm3hjVK_YJuR!m77!8&`*bQ;lXoM`eRvhVjzb(>Joy*ORMdT zax$RPoa2x-@%AUSn|rg}b3(G8repa`FyDv+oQ&~kx`i>3x^kC$RUBSo_azwjt2eZW z_2nne#M*C!45~CuHb}_wHiKxvaJutWkDYGa3AFZ%g91tJ^#^3nm_hL1D};Etmvif! zTB2M2c9x!5@cX2h(IjeUg#CXNx6{u$Asg^>EN5ax4eEctW=;}NeS$(%_TBV9FAuQw z|5(Uz&Nb)|H8K44X!}tWWd>fsa3E4dhaJ)8U@wP#5VzZh=gw~N126fGLRVIXw)cR9 zP)d-W7A-?uDW@DONULW^rDuCnL(`g_;ii7)#%^Rh!?pa)O#Gt8C9TcP{8-KU>^zy| zDcsSN+rp1TeIHCn-K56=H2SMkAl_TCchWoJOwyzIdo3Y5t8x$Blj_Fn7V5?;9JtiV zthKYr+}=A0ehiZ5QMpsgqM_-Vz{R!Jk4-Uq{^AYI5*c-#e*zV0CbB!mX83+VD(wz) z6P`#(DLyuds#_lp_V&pwJ5iL*r)mT(S|zTn0847MM6qRDRl#mL8*NpadyjbfhTnNr zs^W^^c02+@5VY0XLU=v?jD(lVH{0$43ia16*79PB%ryyck)LPdRg|+66h$6rPGe+vSz5^?AW}XUn7?ZL zV*MNAg3(e@_{?cACP0~wFbDXRU$G^TKhdOkj2BY9%muHjjVJ)GTR6M4Es<|MaKUSF zuu~hEP&CcVCW-%-zD;;jlC9R20#`vW0kQl`Q_Iq;XI)z&0Wm*H zA!9R5$xT%)0ns-C5HtLtl0xBC$<%gpJ`g37APfJyB)L7I#M2~q#$LXOGVCv*iyrEA zwVLEbSN?8;8YZT+OlU1W{$AwXjoX!|NNkXJz?A=>6>6pRnboy6XRyK(hH0bh)S8h6 zzYqU%Rm9X?9BLZ2`;`q2cl=QHoFRvBS4H;QvF)i<;em0ZH)PA)?QYZD?cAco#E8WU z4kIxLh3*}FS@OSE&I7%&9)3u531}>w<%HMe2Vk@dK)Egac>vVavlyjS;>Sb0(|zM@ zS|fsTzS<>2l7pTQ+lWVimJ5g#Z!3C4vl}hxUx4xylp7NiA7`nOxRap8(#>H8WtUb6 z2`3U`&%;?KY$n<88zzyHOu0(uYbt;_&T0@AZ(@$q9CLW}fv-4}4m;Au=qtT@X9kFp zHWnoQrn5iHSjRL3gpDvv#$^E+r>CC0oV?3-UBVH@OQuMVg0FH5<6cVgz74FLIGn`VGf;@YB3hhKn9-RO!;G^!}NQeO>$uT(V?0^J+BP$+UWW(Xzlk=Gd02b^n*B? z&Nkll<$w?;0I<`)8h}6lkBi|d9{wz4Ku|hB7f94F3t|F?VLnWFjX7}iY5(8-#sy#Xx{8HL3Z*zI3p!%MT81Mq^`q*s69$SbwMr+EQfcMHg!uPFQx z0CF8Rhkd(fxM(nFg%EzX!S}ciY%&=G&R97z8vE zB6G|(X9vQpyJ~zF2D-viSpbjKPb~}eE$``Le2eF5RO|}%X*>T%a=a$f3F3c93j_W) z*WKKuPfM^f*DQN-JL@0UObGux*Q^UJ)ZXQVNFoUoP&Qrw>y-p@QGSfT{Be%*H6f7# z5}_*0H%|pn0)SWs^UVR=X5ltC@O+xt(mVK#hEv16QFpWY>vg=i6u-4!i|Bcg$B}U- zcw-L;iK{vbkA@%5_#Q9NR01}Pvww+va+&vja}4c10;RF(?=C5Geg__=m5pUq-V*X- z<2~f7E-}$ZLu3g!_a2*90^>=crdIp*7eS%R$Bx|Kp@%>esxYH>0q7SbbWeRn`Q_9jxyTEt zCzY9WN@zz5|Dbpe7Bsq()G~D%bIHC0h_kefQ^IN7ggV)Xga|p&dnz>Z+?CrVJKU_(UsAc4^%sV|iD((KRQB^HmgY{TSs0WCVfSlju)S%fMGr zSGMP4l>f}5e2__$@_&{Pmj3oMF&o2_C3?G8^nHnF3D?EXD$t?g83O2q6TSDvAP2;< zh><@esrZ+i`Z|-UCAOtBgsq7CtF*h-yz-b7T=1|o<~B0XL=T@ejwT_(Ko&wf1|OAX zKBBEWn4H?LTxenI!7)7lL{*+1RECN+e$?8^<_0{}1{-|T28qE+tN#rQH8IIZ+Upu? zwadffp$}W2k8glVn^oRhjWzp98=UKZb@hgB`1JV57RIInb@jVBzHDS>O)WFvy{2Mz za8~kgU8Z7_cn;cl!0@yOIF=1mR(j|C(m|(RgO%}ju8pXy)?$-ZHsnTXG9bMu)E@&W zOWBr0&-}4E+5c1s>E#BbPPQOg=zlj@$_wJ6HoStQagv#FBL5^aiwV~RJ-(mu_sv@T)6q&ic2!dz9@=cs-x44X1z|Qg{R&ms1ClEL#}YK9 zdr@{hh01$|J;VR&yb_hydT_eD5#(&Ximrq9H~_SfouF$S2=Yg72j+<9*ChPHE&sRi z?~d=Jpp}JZo35K1IpfwfNc{JF#YD?RqD_s95C|d)dazoufrz7iO7? z2WlCaIqNLTd`pT!5!%IfR-wB^C>8uga*)~h`u{f@V;MFJ&CVFJ4}TwIG2-UG5UTS` zshjj+#FQ0G-LEw)u%;R$*lqE>-(-J>E0mQZ>ds?deqUgb@GRyfK}U-^&N*1k-69xT zz-l+wK)-aR~kGp+7KQIZ02Vt94G z$BvA~ZYj-CW{AanXBO7P(4Qe`Wdugx2mvBNOa}(|(w=s_b%DGS6BXupeX~cy*g9hi zd6eW+7{~kG-FdXxbrG4DgE17jZt*3z7EXJ|a|&&|T_Vl)3BFF&NeWw(6(Ilvnc@GO`)OwY9j zCBW2@ofb{tezoR3sj;~H4rZd2zbAb{?jz+0RLHIOqQa!(#*=G4i}AC=p0-sKYCf0?rX#3x4U(t^!Y##mIbix^U7V?Fgo2 ztv;>#u%{;7oWV(z-XI@qqL4zvPe{pL+ughK*7m#ESt`t30cc-=W{f?3Fik$a`G*P_ z)!mt{S9TVC{!+(vW-69a@0c%wBVRYbTxx2+|FC#N`+n~qEhAqJ&|fz-AlA#;=f4(e z>X8va@3+TH;MqjaU1F}EWYiG<=h@Le;?;v~2wol#!q4s}l`0Q>PI<&Ab!&pU*n#*P zZs_(Wx4r8jjN34CXCCoOfd;5e5>$o}<8>K2xQO#!z}~@fz1nOS+lB$fihP336nm3I zE#C#D#zboe&7+l8EGWE+P>m5ag>Vdeon&uX>^{3TMF}3R*Q-T!Zk%!*USXmd`Kjwz z=XDT`9n8#jAJl7_rV)UTh+=*~AUjF?hQo@LY3XgmU4MiiO+hutA;#0m*i$uAzu{@E z#Gl|BH>JHq8xodqT18v;4Na1VaaKR6z~87cI6L~V^d$0lQ)T%#WXas4Pd0mjBTerO z_0b<*(*@e7#at;;KsKDZbN`yyrwQoLOh6J^0oaaYU%PZ1q$gY#YR)j9#D>Fz~_diC=!Bcr^A<0rMsjz|TzFW3q{w&55m7c@C3 z!&9Hc#F5s9>?Xd`ZXn~6h4|4$%sS<|*p{Y{cRbApC#LN%kZB{kH?-hBnTP#IMN<7cB!>Qn_sxev# zb(VWgP{)G#K0Wr&ZNh?~WEXv@61f<`DZL*~o|F}7kvbIz`#-R*@o6wz93?)vC&a_J z!L`TDG;&6D^imlU*smFsr1bt6SMm>GBVaj|Yt;V~h9F^Is%SqRq^Eg0!0x(Vq_!B~ za1$3Jw%L+wOFt$qSB!FUeH>(HlsL{#=^S+8(9SutI1%LJQA%jQ-4N;2*52w9qOD$2 zo4YZ_VA*aEn@u}VpPDX;eNpxM>fVf*wfNz4_3K6jGqU|v0ZCr}tXtdYw0=5Fah<5@ zHdAZg%_L@f9<9fk-n;d@2j$Ewd}q_#g}A@{5dhe3DE3K9=9gTAt*u5`+xgI1da_d7 z?$02qUGP!6m&1Qht3h4swc29MG+e%Al6m>wU|c%JI=O*kV&&~y&uJ>z?fQ?$|FC^i13}0Q>yYb` zM$B&;HK^2N1!FprOKnTLz|FTZLa_ihng7V2(oW z9LOB54LnMI+mJ_YOR?o)uHWhro8qDH=F-N{G!pEraBv(uUhY6RvN;deD-QA z(%H@D@%2`LZh;|SAl)t%(xwe2EiKrN3b1G%3UH-=pt-OsXcXfpbmQB-upx<7 zx%aty0~e-=%^hhQ{lmRhRQhS4yNzDjNvBleFJhXeDW|M{Te6Fji}S14;g`KJrz-R< z(tA_$r>%y?UC4!_s^|XbgB!Q{m-+=5kOp2m&CQ{ZWx}6;0Kxag%keZ8CEXXja-nt5{Zi1iCBDXg`Zw?9}2O#CJ+p6x)aTDs)5)uE8ue%6lTv zVYhZYdhc$Q^`6YZuNl;cA1^?K{_SXRT~$+|qe7MqNG|-#>BX2;Czm{te~BGD9`QL7 zc8-dTd|12nQ{3t`0r?oCzk9vHT8$-MQPi$_R^0L>KD55k%*Wow+nq$tiouui70Ggu z8ZFK-X)EDZ=$=)fq3tT2YgJ0tFGL3em@#IFJLkW6An9kl@TC8!RqcDaT-$W9sZ|SGm?K>QTf@J+vTVG@dm}gvbGP>f*gkGP^si zPAAtStn?_c2TZv0)N4$OjCMeC>511^wOgY|s;M1G(t}HQtCr<0Wafz=mw`O-97#SZ zZrd~;^&?xw*$J-eTV3yAct&QSh}5P|UItw)69hPR9!}0+<9Cq-4Tj~CJn?ztXqg#0 z-h?$Q3H@dhsXW^c2|x8X%Q$#noAG&r`%HweDIWp0m;f4{Qc zz@jdySg}%_V8kzP4)q$lwXL02OR8lLNf=72<|M(u6g3HfB^_^ZP`zT_vwx++mBj98qrYb2-W3X6AwCp zn%6CxqubMG*M<>mbq?%9sEW5tzA z;5JpVw~fWr*x8*-nQEsbF87<{2!f_%+;atkCCsBXq$lFl)FnS;L6D(~z!>kl$HUHb zdOlYwgWSHO;9&%0O7}5{j!XoH?YZ4Lq=8HaPUg#Lvbt-sTdIVaeJ9a&^sXZG{sqX| zyNd%es`T29VD#S%j_hGwp$afZ=DPZL`@HvH!H~7r4~gz8=}C$_F)5laxZuN(1wCtZ z-yWI@vznLsqZv?H+Y+^m{1`jPS(O>|dwK54h<%T5w9LKu=6f1*nhI<0WA7;X5Erds zwQe`;NUA1J(BSZPGFY&cWpF+T0z{qTUDK5;oraTpb7(cM<2=241h=8mh7A+}pRMQ+ zCv+|jYo(r7?@Tt0+4CC`e%GIbpj=K^?dNC|s7ZBIF zMLJlK5<-8(5T?UQwWiw}-y!it!;Z{W0!<{G& z_{;7hTwD;DT`-p;O*StzdBERBU!t5cp^PE;U#L4EoC3>BKRhct{w!E=j07SE#}2aL z!>nzV9z6A*=rrc)*A%VHd9#bs@1(H$g>6$(q8Jfqx<^7vX+g>gGA4WaQ_g==f!0* zpD%7PRbf!A1SYN7gb}d7l91d0mYvWc65k5eaYz_*No+wRDqIvzXB)`jAs8H6>Yeng zrCz3jf$%-;H=y>r5t1hYZ-g`AxDRX|nJktTYLrGbu5UdVN*nAqoH4>CfHIaLRMPAFm57sb8N*g*{&b zUz4Im*WI%`!Yp6KWM)?eHqWJ~5z~c|7-ek4H0AIfT-Zvwx*dLIWYu5R)TSMl!-6D` z{UoQ{J#`%ng*@5{&oea-m-@<_D7(5^$x2|P$o{`Gw6oRNVsjbw9}Cc-w(rWa*Yk?| zwI+YYH%hnO|16Tb3IxGB8TJQ-=@mMP1(#h~G?(}lK5a!3w%&3{f-qd9l`LX&`A1e49K3<-zZ|`|9 zATTwwhe$5wX-BXMK`OHG=rr?lA}c}hg)n)0M((9cc%7M3f7V9Sc7_>LlRz`J#Pzn^ z@AQqRt%O4-g(*|EEQntWXY605AfA~e>|i)`CU5e6mYogZvUI?q(ziUWhpnlylm7cI zrfCk~&1)4u{Q;9oZX)CF!4Fjd(dVxnt-PS;Peb=l;pY#k&QI27Z-b^c-%o+wT3p*I z#NCx|-mX95rX0Brh=VTAJhbQv3HC}$beJ`d<=9fd^I7poH5{V6GH|d>PJGsya&d;n zNq--gNaYdo%#H>Ks3M!dzK%8I%8J)f$}ELQelPd&DQ&;9fVp&>f$L+giK1x3f{W8~ zJ}&3k@L8*nk(LVe%doofdnG{lfJ?xXny;_(ou!^MN<3jc9;x{odC2{q%Xs#bpyKz{ z5lgzx;7xcmCSb;AZkImpL1Cw{2*WWudL503RL|)OtNab)8p<_h{wa$|^Y!a<`zwyh zuS63*$;L;|gth;n!-60a6=IXYQY022F?gllIV&F_`R@>C0YJWJ|0$SZ`9-7IIL+Jc z>cv#_8c&rqt~VPx-1T1Z=aj(D7JZUJa~2WY=`zi87Jr1jw(G3Uzk@TqLUFS$VD$BJ z4)?sC%5n}TLS>jSA8E|TSp#V9$U>cRdruxxZa(Q43rhM_=~T<^>L|dzxX>CS zq07n=Ur-$|IpiI@ocDx-DT-22V9j;;43DeO$~K2~<<4lVQy78~bz-+(82IMBCmjGs zU8=TQO)X=Nw%$6`X=`2qf9omtxQMp+Nye@5>uxHvh%p&u^*P1b9fV`@_IJ3e1oh64 zbuXc6jMA_aIe@|GnMsXVQ>}V>h3E9RL8&00o{3_Dx*>4noagb`nr|~1t8SOsQ;2;# zLXz6K-Xqv$;tb3(ZM(0^=c?Z*rjZg(Mw~S>2xQ}t|3zSu?8nfI2uzE_0?hcsNCn#T z^@F;By|^1Lr*}ik{=e<-&zb{UuWB=smX+JZ9~lq6R^QB(twQ+0b1G z=i_G0897q@%9UTCG-;pKdwqY7{qV}K=X;~IW?BCdipRk(oyErG5NB7KRuf3|SW#ew zGL_Q5kjbdA3=}?N1aR;UH$I#Y5Ts`{PzZ}HWwqnO_$}!xd7v6{z2D5u{IYkE8x4|- z!&sgqX(6d_jy?{Zy>c7Bp7-o!K$S%!gP(jI%(!Pb8I0_DA5DhiTvFeKGE4b+FXuy= zGlMI;=zZX*`8DSFP13gPrO3yBD`7~!wX?Q+M10!zh$m6&X{%(uUpr_YJwnCm_DII; zjX7D*gqQ~JnE5tMxR>xCLLjCrP~@+S52p#Lc=XVPVxTYD^wrG9GgIz?sa<|cjEzO` zFDPHDrz32 zuKH0h2!2KOmo|?|ojy%t={@$OBdoQPOZV0m3Sr$i9!It>lZCmc$R%Im3Wh} zF3-fZE_WB;#;OMOdv*C9Rpfiwzl!BKa#q?@S|;e6{_3G%iKx&a738R-MWsSbEIfpJ zl7RdKwS!0tIwhjH;IG7?qERs`qdsBp=)WfDquR2`0p3h^84qi;q(TBt%{W;z(;|vq z0rOJIG$N+TuaV`5Y4RFb`Y`qkzWH_%!ukGQqDW&u8Vcza{#T5M{Hhqh=gnI&^bYJ4QiJ3^U zdp^SOsf2}$w&Wu3HiYE64>md*6&{4{!BhTZeCEU5!9_~)9YikdbIKf_Li-#5|GHlh z?Bb4e%&nFuvmdIUS#!1b4E{Yt0s9xqT$ zs@3mUll_^qdBHh=h-k;VdeRdKLsk-5yq{LeG})J^Bm-zi!4ZSIofnrTP?Cj)(i`u> z#s2;Eu?%%LWEro*h-l#g%gWrz5JaNiZxNekiAP?Cy+-<^CcYLEei_u53u*-QZd=2; z_qgXtuRREcd3?-^|7)e@+Ifr}Q}r)@`o&!`9HPv0g+f{nM!JDM3N@B9a-zBm+aaZ9DhaPAG)vmF zN=EFk&^mx)F0zb2CnL)06EwGEj%H+`k;IgVgf+`^f00D(GC{A}s-#I6l!0@FB|I-o zv|Q4=%MytaYurmgaww!yW&D~_NoQx(e{>MoicZPq>#eyEkrv^SScA2S{mmAYb+Do}hs%Z?G{iYW{{Ayk5{6 z^SI69A5Nb}k`EbR=4HVJlj_3vN1}V*d33ov#TFS&mEJtUzou4Ez&`%SNicc*Rp(bL zgavH2_P;>)^_R}#s!(o~7HlmoUm?zwCc1XgZn0ti=RC&S;Jw;FVvgake|(0m?9E66 z-t^{Z3zvklGYae_P^e;(`b8{+%v*N2NoC+F{^BIKQP-k0X_RK5)bS@`#C+97%bj^C zG$|MEv$!+vAu9<~_xH_Kz~C64#CmKtL#poKh9z!5TYG?tm6*M}y>&Qi(+N9YS5y1U z#i)@3_{e1|S^La-h$v5@`0z_c%}9+l+yR*d*uV2k^VbfA^$g8;aL&os+xS>uq`qkf z2#I6H^UbP}{cClRz{uOM(yNEb>pJ_I*~7v2*n}9!LH(Ziz?hMnukPtQu=3k zYfJIm#*8htWe zGHp76qP~L%L{SSfXFSbjVyGc7D~D0;h3)!Fs;qJ zQ1uQke(Q}VSv1@2;5i^NTd>ma)xi_ilJIm9{TxWb`&aR4b@YHs+AwJ{c;;|=X-iT}8rF+?$ zyXB|;6?Dr0B5$n3t)-F&K27Hc-<&k4`G?nGpZ8ewE9X3kEAVCPP2%sspbAVY%S?G5 zYS@?Q2>9Yk3|qtH8BK1P#DrT16_-K#JsLG%m2xwkio>x2U%&tXf#x}s=S{E>{HoiKG&M4k za=@({nrQ+RYd+BT7h#=Mf4Y$3&Yao$<@ssE$FJ!Xz-(9G>d}|X1IHCWh4U3q{CAkl zHY1S<{LDRZSwjl8Hw-~uR65f#n7R6KmeyPp>d>Zmqt?^m7q7ss=jl>>SeLb`+u89F zkW(%GzgU3ZUHH(-$~9IkwEm*@TYwt=yA+g6y*oRIH3EuTm*i=HSPIEB3cyRlC`zZn z-mllIjsmqyYW07T|8nEqJpFrqqM%TUDJ6?x7 z<#UP&ZI!&PhUkBYq8UW_wDIr&C@M#V?k|nJAcvyR?v^269F$zCaTM>ktgQ~jSOQi; z@o2_PGqvB=IAv&h)+`taMQN}XtyxZZoZ@!+*jv?s)F^5$lB)@a)T|IFf9PfxfpJgb zF<72l9#9uzo0M19I5%-J6BoT^iS)F-mtDBKREh`j90}1`0Gkae6J6(NG5|G$KY$2< zy})H2af)u1(}l|cmUn_Kz5>ZQ9&eGanOonFr{f=dPE4$?8{;d!@Au`cZ)TtG$N89^ ztWWdz%jMYE50q!8`}64O?e6;H;28hsWFPze_WAD=YhT2S|akDRd0oT4lFq z%WhFd`3~>_4UeZ2_V@4mY0d{59u8OuHM){W+LDeuY{NNuiXnR?keFvjvJ$aaN?{;r zhO8T)+jlwrCp^MLk<^0pR#zpTZL(;$IF2sWN8bo5ae=AS)1tRNH6^@4U=xM9Hh{}T zb$W(`qb?T7s4l&q+`gk-nL!#K4sOYIIm6Q1P2rS{n`1twFFnO`R<)1JzU9ItjgX9C z@a^cYwlUU}hBk>!8_rUa%6G>$aMxk$RmRRib$Bf(Q7XF-Pw$S$TBA}nMGCjvG{nb8 zh8b)?c@xp!cvMB-lCQNb=n7g^#Yc|>E*a#6^qmwkE^XO`=xbGCfyEz4irqK#1#N*) zc+fxEOJ>H+l${{dG|SvvMvs&ojh$-MEczge9qb0O(cwd70cO<*OGB^*4j*8jO0>CD zNkXyU1@k&6e=x3NjsDS`tF<%K1u=%AiRy4YC349JQi-m7*R;J7fGqf`KXUvEjaHUf;hus{C;o4u=; z_}#)YKR50x_{2Q625XDDoeX!aBmPOmtTw8Jm?cD*cT81xmCQqyxQ3oeP5V3K#|sZB{fut+3Dz@jWQ6t| zsppLO2QHACgx1%tr3Y={fh$xHpA}TEDomx^Jd_55R{T5IFU=A5noPT*IexonrIVs` zx>!Y8fZs%WM``8BLmtXMa)Mk;c1NuU???>zHvoq@t3Z$+KktDyV{9-!$he@*J@=XM zOFr&Lqvw#0Gu<5*l1;~xb2-gExlQtzqu%5et@~1Dv4f`qmtFT*Ik7VWP4rD#!Xcn&|B@onn?iSMXWlLqP3O&xENpJbXcjyL5FRlS@zsUl#sLfMs+FR`%Q(4gHYD63&n88V)L;Z)EiS^{{zfj2Ns?rf&UVCDYog zN~H^Md7f4lamL2e&zxN(MwrU%(lmIcK>}iFwJ4{PG2n))wR9P%cMtKPy(?i2vWr=2 zH6tSjDv?_#(qrVR$}#D16vsa_5iHu9QwtBnvrR4y&4uG#1iV#N{*j#aZC%QI$%A#R z?^f9{lE>(f=79yFNaZh?o8-#Ol%egB<^Tv;BkAvJ z&YOZ7A7;_quGR4=yr(g^V|DN2E2r1ZOu)b=SNzs1Cj%1VbhUNQmQ~Lt{hq+?a_`tcXwGt_3%6vPveHS<=1KP^W(IlRtcv0Z^T8 z^=PONY5X+MVES_E8(sRHdNblURy4n7s@(?5xg9rh?DU2ZpDKRp` zlPmhGiEujkt6(%8oiv~j{rRDsC*POct~8$_C5@s1{5ZdBpmEe;W?{8+p3Xi9@)y!5 z^TIO`d@5*6XDSVUD9IMgN7kgXU9Q0j_Zg)6thsbI7z*siF#@MrC7~4i&C@*Uw)VKo zVc-SKlM$a=S-m_j`ZTPapirMnPMD{mxi^O1?beh^N$ktj(YR0GmmH%8fKR&a_O}bm zC*tWms^J3B!gSGW3w6Fh8ihgtpeiDzBcA%lVNVg*eC*Gz!llD|<0p5-WYQ&{vw^k5 zlREjI*+|krP}$I8k+N%IDRa$#pdq5$3@ntxHQ@)REKmfhxfjt{U7_X+s|ZcHwG2fRGXlX|!85F~?gXP*zr-8fLaX0KlJ9TfJ zW&v4H*s6`^HTH{R!kyq8iGenlOvU$CI*isYB<_)JGakb=Yh-k{5`1q@E!I)pT5Hkm zZLahlo;Htb0J~8B>yvmwkX+hc6F!n-@iEb%Bunz{g-fxGugA6Bg>3xuc1@Bnkp$*4 zDl&>T^t+~-Od=M@)XP7FTr2FVJJ^=vi=(@bsrQLnp07t~o|q&|w1j9krLKi#%bY%E zY#~eHr^=XBReT2(a}Y6%)WXWbhb-f2<6>vU13{J`rLz33!QRA_$Ijsxn5j zKB`Q#ghQi?uk=dWjYDXsE(5s7{x%_ z8EgU0`u7ag)^U$%+C_>AG^GZ6y1ggzCHY_Xa{zsaP;p)|->&6mzP73%n zI?XY~CUvtO$eujRX_Umgr%2Eacd(KznV;BSH6{nL>|Rpg2g%evvJJJrb?@tS9Kd=9 z_vUk1Gv4`IwAo7P;$EV3GsoRaZadMDaWX-Mhyy1k(8t|oTAzn(!YiHB(R?jIrkmF;wu(-K5p@=x#d z1l}Ab30nP)M1nenX2Qw$Mx&y`>DM>iOJ?Oe}y;qbLrJqXtce zEP#NsYGdd+%M*_@28{?d6XZ-=tGLMYK*gfgdrLnwwtlKF_a=cc61hZr^P85C{6mP6 zK3Hj^s8>oAvZO-KSaroIEi79X99?z!ow^FwSaH}ZU`FluMrpv35f9Oflb*4W)Y0k| zcG|DiL&Yczr{M!SYe-vBm#(To_Z1_&i#%MFZ5~0$c*lV|c`Ip37cNLp3ox;w$3U};sF^l7vnGd9OG zv##S1xp8rDvoMxUHL0rqr4>)+H*S0PbO%wD@%xK2a7LPvwec7=?mG_7&B#%g0S_os6GEVP%{C8nSa@LR%qjPj3MWgI zjAv}YJnX@wWI;C)$qQT;+sJnI^#$;v^_RECJlDDka#b zJ=DM3Jt4*s!#+`{7Jc>dVGpl_sU!vSQm4g=5s-1>U0W77Add=&LONhIlBVU?iKz1W zjf|aS=50zy4d;7&{~TR9toY}PyBW)!Xe&#na~AhXBuUvvjVQ()t6-s4p*9+PSwJ`A zp_m`d=xopXpX$6Rj_sgH^|98|yi-k+cZf5r=}>i=&zfwn!BLO@RJE)y0OPKq{nK4@ zQ-Wsfb#1ASOPB;B>n-ua3PdoNv%C*b69I)D`bCS7-P(up4D&R2NgL58H2Dbarecq< zIrZDovjvpQj*?0fW3Nq-vxi>!KKxoSeLF|!T-Ej2Jp~0EDPF<}DQfaf3=f5xak+fO zZ3EYzHzo@RtRCgdjYNVbzu&;3HzmbE*nHMV_nMYGBY{;lwW8>A?M$&;b ztJP~e_21=;@VD%>NS<|!MxfN+E#Eh<&q4rfvt*oLy(quAMbzcy`>qS_W;I~L?9`=8 zu&2D65k*P3s$bcfxBF=5^z6<ih32R(|-V$cw-hLC+P22ReD_++K{IE)#Fy8)7jM#UzqBYu;ZMvC1k0g?r-y-`(x`*;{x!a=NUp*E!Nxx+Z^F9RUuj znBTHB(l^CFmS?OpCkaz6bfaH6v14Ny&PtStpjoj#{cBl}9Vp;VIgb?m^_ADhyn5a? zr!hX}eq|Y6AWhbM`dsx^+1$3Zd=6$1468x&R4-;}psLM0v5vEnmXq&+amzcCX=S>v z#@seJ{5XkN z($RjKilvqEA-CA`@Y+6Fb*o5U5j=0@Q-5PG*ZzR1Y0>ZQS~kEe9(N|W%4@i;OU`-{ zJksseWX(Fo`v?)I^7*$GE@SzP9K-E!<*hy`O=&1tXNPAVNg@A^T$$I@V^{r=HHsY5 zbSKX-jpW0L(C-bq8-PV-0X>?r@CeefXf% zOu}AaX}A*4H1NNz7e!j~pon?cvNFy4?s&z~JgYW0>YGSfA)&t^Kx%JAF_brwaeR;< z^m?-HtS-ekf`osKmj=ej2@9i$>7wO9L7vZN?;{@PlF{=LNv7)y%8-P_-asM>3zUIj zL*|!p`!Rn(+CK*`GJ_UsTz@(S#G-!)yJNnhiFBv4&tSGWSEy(P&tBTFC}{Yd8Jg-3 zIxza5fFno@2x|Q0ilUS&FPSrhY#pLeM2UM|JU-TtROLM^d-^>G4rc33x*k*RnjcFo zA5BrnHb0dmw+~#6yEl#HHA#w1ORUrqp++EJ&k?CRgNAkbT^B-lzSLoc0(;cwDnM%| zpFca8%m9WZH|bxeZ#<_>gKe7O^BC`nZ(4996gGfY!am}#S12aUsvSFaiJA<;Y0RRFs|x1@?rLF&9=bEa05HaPkZB($UmZoj>bjPbgdN>sd*)YVI>)*|C<`5KPMLFg(}XiY}Odek!2Ip@hgM0Q!0Xm+_KD18}} zgQIH{&cSAlNz9bt{HTZ0%j=If$+u-loFzh@Xsf5}^$<-DH@ft*`*9sZzn zI%D&n?ma<6>@TEa^D@<*=VQt{QgDlOqd!cLZpuYdxDGc*;lGHd%6}|^rm6n?o1i=| zJc!Qdv)Z~AVGUTk4%&i+AS}?OZ!-9HGi!bTJEyk-l$-l*~`q$W^WL5z?r zvHF=X*{1y9A3au;;{0I`wKNHiw!NN^hvY6Dgfd1KEryl$+F=p{?wfFo(t{HR;&3338Nw+Mt!qfK~1BmY#$DLBn z1&Ny9>^}&RC%0GIn5yT+UrZU8IO_8BY$M`es!NDi&sI#@zP@u@;3w#{A@aO37Md)! z2CH(D)U~%)9&X&&Op@q{t&sJasZq^SJ80Gs%Z7W^7@yTNrbsG6?d;<U1 zHdDfc-*g8J_@=L9_r?)TBH&CdC2AM{fgmCfQi}-2olcwc?%6awviweFx*Ihp=}Qu2 zD4MJA>jN<)wK+e@hjl7iU1na|V4IS&3&eHwF3rRFVL6eJ4U0&pKj;m!vcqc4`BsTr ziMqHp!%JUrH4n$ut+{9gRK2}O#Qyp-x>*T;cyuuV7O6rmKnummL}Wuy!v|=SKus95 zMco<|0VDRt_?+z~LV(HJ$xaGn7i71t7yvbBGJxtI5>A1%E zoCjApZW&kYPf!odyuDF%C3O=4rfyBHx>pM^URqaumBhCi%h1d|RSOX-5eUZK)@Ode zw`OgBNrZznTBC~sD7Wo?9imQIny@m!ia^W5M?K2FSh>fE5H^ohIFcck7FA0UsOB zqDo@K$0v=D#b8Cl^@?<+yKSXL{WwUf#@JT<@(g^*F(q~PIa?8^5ygH~`xZqv>E#G) zo@iE#&~w-ejQTL5;Y_sIn^cB>?ftD&cavoirqalixo$PeEE4J$+M=xvz=d@Cve4KQ zD69P`0}U&f1!gJoMyVO77D0pnZCY>r&4^mq5J)J~76v4{fNhYO(%mpn+cwf|E9OX7 z_=t)!^|BhO)GB;&AzjgLN=b=sfkeDl2PmFP;`-MerwiKI3<}phVU95z`2yH|v%w&p zS;25l6&_$K+#EOAhBl!~hdMO3l0X)N<<r{v5gsInCx;JV$# zW>bcbx*nak$~iqhg5=vbfWuh6e`45fe*yLWx~&miAW_~4=a^_BO2*e`{(6JJo>-3o zijki!Ud(PPsfeZoT05ecj#^u*0tKqB(crukq{*74-HqL@(tG%W)!v6^h_~5{-hLq$ zeHrRic2AX~`U*KOfJIwr7&Lja*MBC_`Y8J1Jd*Cbv7IiNQ|CZ%9N{S$SI%yh6hg;w zmbB}s>@1=1fa;wOfbUgSGCs4sT=MRUa`%T~zY@YAX?cBxf}u;S9j8P(-9vg=*vn|- zlWRpVW{#?p*#x1UHsdflA9P#VN$cvnB@7t_eNUw*Vya7%znysnmarwa43=hnv31VCCz=97FIFQyaUB z)+^!N>{Yn1pLT(#>YXQ8`Xr)660_Y9^r2nrCDAx*<^(cJcUH2u^F&md$i9rlmKXP6;6D_-$ZBqxEB|eSckYWGU`{TY&(nT)IyqyH59l*Mh zL|p9-1=8Xsu&nW5=o@(2RgLHqAe6bLm$&>TZJ2yrsj zb1R|cO*{vQYDFT)24i0F4cJH!O=|a3nJt$5HH>(*W)rhOfPbHAi0`ise2yy{M83Tb zg`r~qcn}jngBNtuKt91{WxccU#}V~=yxt1ILA?!DjI|a93_-ICV)fZVN`8ReX(jHH z#vm>YzBuwD` z0K8D3lFY(k02%DUn)%ty`Jl1PG!LtN{o9f2KY4n*)w>~!&~d}U{tsR4BKmQvL^SXt zP==e_9>1QhRhkSJ6wc?e8SCHJ=CGw0P$CG!-IF&J`_=*J?T^`%ydq>vw-VN)@-~DN zSw#zkx!PNV325$Qg}*p7Ne^}>krJDY?oSwm78RQL5URt@ zUnMzd|E{}{1}N6Of&?5H27bt6IieZbW2a?%1~Fn~Ye`lhkH>B*&x@x^UB4DHl`AZ+M7#Bxd(D*5MLf-(jWNR+urAvgAb8^jGHk-Gj29@^9%-QxP zEd?M)8V)wF6dEt0xKvljBlZ{;e)%8OZiu$y$V`0T2ht~7=0SpCiahkCBWUND=`Nra%^xw!a`WOzG}pK9-PBS z>l+4?DtrT2Fd6c-Pw=Z%_k2Ir@$;$%MFJB8nyH`TFBPZT~D7_YKiVS zaqT2ZXUZa@v4=J!vLkbuS*FO6L5N=>d|woF%I!o}NaTb)<#u)kQC!`~XlrUy0@nm0 zbW!8^i;Z>goH$VRM)vo+{m`G4ZF+@q*hIfzAy&9*GPmZoXOBV^M^;xl_G}M!I%fJ4 zz-A(MGdQPAgs4EOCEQV=_+xzvCZ-*ISpI7gZCB71Xvo71w{W08vFe+x__f$O~`Um+1{+Gw`e8xu!oG-pnNKnE#(Mn z$AW>`dZ4ZZ`vxu@^~x(Z(e8ZMZFyBaQ_5IIv z;sZv0L5$aB6Iq_ex|QBL?b9)wiXR`* z6MwCV)2%+?d}s9)9Or{cs7M*6^0oR-9Ht0j@81NR{Ok+sRE1%8KsRFXn=&*Iq|aWs?zOjWh0><1W+RTgMS?e=87=L`wT{>G#Jf1c=d0R z4Mh=BB^&TLy3XEnMSP1K+BAN`M!v&ldV(o^jiIO(!{DqEcpE@*n;TiQXkq*os z$qQN@2j>B%x~9>ft;)2ywt0TYv0ssiRD$vRA|irKOo799Z=Y^X?%q!B4nD50N35^w zQN7-;?$TFYVOaDA^X3z_@FgL>?CNzmtN6;R;PFa9o9MFk1wU!k)~C=li#F zm~=Y@xAABl$QZpAS51z+@|ihQkI_)C%f+pNV>8Jd3K+ne0kno9z^EnDe;g|(@}SHN@X zzWY{puswasaQd!=Q+f#9=7h63<@DDsp-4Km%r+O*58zw60|d%dy-3dp@+mAPL!k_~ ziH3C}E1<~KrD>ehVLaG!EZ9@D!ciW{*``cBIF#6euwD|iH0sbPrcCTTMd!KsZ6l)U z%6b{5`7!LdN?Wa==RCv+sr|M&hic|$BF+($7$sYPPs07wlcu%V$m+47SjdZ8qgwM@ zvJO$^K|4dUR%X{6zGE7*@dTAo&`~LtY2r12+!~r9gC}deJ}K2fe8~l_jJvYaFcah- zZ)>V#nk5GTes>qhU(FlC%c+Pm@_7LoLbaJ`jnbxKJE90xYEo*D}R=>0N!&wYb>J&;r{I&9+1 z>EClVu9L!kDy6&2p=1z`yGJK4)q|j;2s!gYS#ec?TVD!qTRK@MOUurbCX}{{UbkcE z2;)3YRnkSbcq~`)qa>`xuyZ1bYLWheqKcB}r(kt@uS+*#$84VYBCpGt6#18E`Jz=G zA;E>5R!!MP7Dj@LTrk}%`D_kb`80g&){`(VeV7!rp$=Dei#n_Oyx6s^U{*;7=2HLOKDG z$vQ|h`7|ssC9IFBe8}@(j;42Xa&UF8?kN{YvsxE;(W@oCJ07)j(0e81FKc4wI1 z;~%BG;PrrI=?rH4wl%~8TUUzZLwp(RPlSaLzM^9U9?{#;Whr%#u}6gOETC`V%7~y; zXrfb$4A0^v5%84`H^ByfHi|GvRGN(1Eo1ndOtSVnQ-M+uCsRVAgjQM~%?U_t9Z_d z%X)?orrv6|MNof^2kM;J-j*xXNxSXoB&r1mI$2l`TP{O<4P^=rgxzM+k+Rty*4Ap< zNKe$V-1B5g%@?%8OW||qu{~YaRfeOj$&|0qZj#7|%!(pY<%WkrL{qT@l^a&3gMo_=~fnNEc@=n&Bra$0kYGr4th+ zi~{$(ZbRmF{ok)PWP7*ZE?^|9n`J4kI;FK0D+HtfAwhRIyPdxxTl5k{pd>IDSP~+W zEVq6Q0J6KJ&1?V!vz<6i97w*{a6TNug>v#EbkJVbA`YoC6fim&WS1};L+~U@v;Hnq z#tmg(Mxm3^-_haNFdRwOBO4933bkV0FR^%jl7L}dZ%7H`uinw$DWrxLRA5DnZ7Rxj z67ieOG#hk+SFKSbJDOUHS(WCIYArvK+YD!W^>j8gegg`8d~Il*Gr(r)Lnfd(~{wpuN! zTAAT%kf=53{tId)1_xcs3`@+57>Wq+YS37Y?)Mx_*@qI;b@Vc_-k!L|qV+xGR&vgO zJN!4N4wO%y+~*FMjD7fe3Wv31GO2BL)>sPDcO7+p`BmMkaZ6qr^DkK7$IQjbz@NyN zN7;~xS25Egu>*Ax4Wr)E>2pI8k+TV$7dY3}x@&EnsA9@X4jQ!S-f2NRX;<6#kC~2; zo_JSnDrwH)P{I+ob0l-93(biNp7;1+hc9e9C9B0rKm&49S|v_vY_d=65)(5;l`F=| zY$S>A%N=Q$@%PsTy&l`uH@QOn?tOIVmfDvcX(GlMizVj;I*LB)0DC7GWXr7+3tHPb zwRXbqd9~1or5P^oC3IB5;vQzXjurBxNv$5e8_PDIFSQ9-ud9z^E;{D%rhJ9JKjPz-BYKKne zWXEE=Wfg6rXHo4#nJ`}3`Mbo9w6x>|5Q;z`5JX2KOFO)p)W{s2l|<=ZBpwls zyY{Y_%|u!@_q&eWP~MY|j=ATJ^yq?&Z5k7@BymsW$JGbJWF_qZ6pU#AcCSB!%tjl& z+MZJMK;4<$<{m792~tjFwBNr5+%2ng_f3z2bAi{@X&k^wR(EY zf~f6|1abw23HK749k1c(V)mM`p`Bk0WKmPAC399mnClYxt|Enek}c7E6{!V;nO3sQ zD0%3SBM(0Y7|xz*7-}f08NZ;_kevB40iiWgP{!ZNxT05W322Q!;oP~e#rM@hK+aIu zC=}3z!om?`!!f=yi^N8h?yq;G*m`*?|GUcXp(wPBrLsGa5u(_o3 z;N!!0r?4q;B?6{UvC@RFEcP3FLuv8h4!cj&<`+7?+ms>T8KZZ|J`84xk>B80un<&b zg^C0vqqk+Ejb&M z(q=7f@%XLSekBhkC(O+W7b@j=Now>;VVei{w||HezN-6@Wpo!x=*02e2sp9-+ucGU z%h@>$R@OuF2;T6IZ4Z>tsf-Tzk~DIMkWxTp3{Sd85tXmFJI%2N0~SS2lnGo*BF+X@ zbjllqWPq@lcN2=xSes6?7M1&TF%j;1Zsh)VZy*pXdX~XyiazT7YC8b1&!Wg>h8P*E z1dVA5_D}H6V%(91{7?BJbeQ+-Vo#k0Jk5%FkC7HCiu^IpNU$nmlW#RqJK(2JcbyTLKYA^s=4J{)S!pvubj%HI0{_V`*7nEGvD_y2FK4shXwY{P7@F$e$pJ z^QpaCy@0oE!odxn#c#3vIe$$Mhd1WQJx7^^wU@eQEmLrzKsH|!&TPPkuRN-FNb*4z zeP1KzTKa?TK$7u8g7x57BiuB=cY%mfA0tu*=Dui>JqVN*35nePd92BGBcaowcdXgH z(V7BR9GAJ+drLD771mwL#f7&E0Y3-&0<%mphKg3Vo?w>m;!>`&09yRA{7*9sJgT&a zW!B@|>yQ4qOiZ4^IzV$~_VN)On5|=7G{g6=$M(E>Xe#i42z%F}%^%nLBm(-b1vO;| z=reduRhuq~8h-BkZX}U3&M6k^cY4~gUW;$L1cmGgM$Q%$j*lKJ-ih<#gr8MmYmQr#@moiDlLgS(1wd;X;CFyaprqb6|~V}pR*04J;N*|?T zBABKoca4c<^h@n#=&~)p&0Xi2N6*|>4~_V-nmM`sT6t^GO8D**$Oovj7HTy z!_N$EIe%iCmxy|8|6*^HMunfrQy^217Y=m~DY%33X~}*AnPV1T{W_G$;R8+I+9p!an3Rq4add}=InD>=nBje5r4cvW z3S#p(*^t2TbPqY^@FRHI96tvoAp58ctQF-}CM;A#-DW<@z3!hn4o)LdhrULi1o4S1Zq#`SC>LyIOInc9>?(V*Y_`jcafRYS+B z8$LqmsvFya097sX(gjNx7;jCRf8*fIHt>0|=kaQdXvDg`hY0N&M~sCrW;5EPZXYLI zs1kXC`a)-1ch@9om8>?wdcqYt9xYhn@NQFoFKZwcE-h8A3>Dy$Uh5tkf0`9>EzNkY zb%5ZM?b?}VPQlM{dwp(gMtExo>M?`RbG0;(4W;DcUKb9f!U{agf4Vk({Sozut>#4J zZ)6$^LAygZ@$83YU_jYP zwCX_THA7W0msKhB<9?C59m_lNq|)nxly#U;d68F2;Eo|8&k5NXgbzE4s8&rX%QvMo z1kowVEb~D#*Z})9)S5XMu?1xUTYnxAZf_x~$>I=QX!lChc>Ln>`mj60KN;wG4tW0#sVSK4XrNT7132Tq+xwD!&mK{?yY^%kBcUZNH*o0 z$7ODGBSU_ZilVOS6HHB5U}yeQffsPNJC zP)Q2BzkE2+BVsy15ShoT={_+3XGCHY7#!%lk4aq2w=27U2XAN_VunysERmfH2i{pZ z1o7NbVcUAWY&J9(bJ2|`hwtm7DHv#Ba5PVk%evuYSD4MXW?ZAZ^^Y^U(8n{Kim(vM-i%mwFL?I_=78S{EP$rd<^W*jVI7=bnK`pRea?%TU@_U_>pvo$u@g7La zmnbPZ6#O3luNldOJ5wtxmr>7tjrWgV@q``s9Cu$vW>>58k9MwnMhu} zU^-0cy=q3F8Kc~$Kew}{$T3|L`W@Bes%CQ()zIAo>+27c_towjC+p+&<)wyR?0ett z+vOw2=q1*d_w{~^|C0af^5Ed7wA1QpD--z&@^zjw;%k!X3v+RBa4^$z z$}t!5Fsa0f{QCbWd&lTXz9(EbnM^z}Cz^?E+nLz5ZQHhO+jcUstrOce@0s8K-t~Td zKXms#)xB!(?$fpU*|n?cfoi#D9)NN_ouYYiK5FlrmJrIEX>F^|8*<7G0LX~d=RBDm zi*9VN=fnZYv)*f$#Q#LlbSoN6F#AN7mDAoIOFgu$+T}P+q?Yyaeviau)BUmR6^agP zCfK-wNX;S?0jJ#8&Y}vQ2RExP*r>YEkP}D20F=(^=U6miBKHTU)>*G<{-k?kAB+&| z>OkU*A&@IPmFBAT#f(Lx{Hu=OYu04$1Xai2tEJ!7@ev$9OK@oA`@p<~H!*pR$@AM3 ztD*HhK+j3s=Q=erAyb%ZuNVY$a0?6)?3M{UA|>5ajls@y7>!|yyJ$`Gi82AY3vj<6 z%l2LFI!;1lUlLVJ(*Dtf(l4F1<8Jt-z`h^yo~N%8=#hu&jm)rPB^dw~G=dcGHi{G7 z>sqT=f}kW8wQbd-$5x2}#^~UkjgLWS?c6{UOK=%QpBlC)1RAT+m@*m6WuTNZ7pr@E zyjb)YiP0aBpYqN$bj1hN*M%D2R?N5G1uxV)*q$G)t6?k&8((mDAb6d=Pwpgk?YxrWxi*T%dfWc6^#^{jn)Wq^C%6drF*k z*C?CU_L2SiaDag;Ez`QY*nq`%9d@%w^=wmO@spAG+lY}rA>#uxtwVw1fwS(nkILJ5 zuwUzh?>1*YylDnL+?BHZ;9~Im|;DSf+<~32Hj6m9Pz8_DhBUS43 zDH>Wo+NyIA(iI!?!w&R8lcuH%8zvfDW2(+Tq z+j-p_GLiQzL=>+k3!zDvtsRSBd_REgLb^#M_>E1d(5-tyVTA?yMv28;+o-1ZF1^+i zkwTt@ZMaE=0b4zqGDzt)u3#YCD?ilS$sVo&pi#zmf&5wawiYZS=BU2!2`o{Mf6CMH zF(gd*eSS@44fkORLK9_5BJzL@Wzr+v$_1UvFi2Ta@3||02JlzujjzCU%U$c}NzauL zPBCF?SP9QX&;Wl_MOOVB4ooW#o5PlrOZ+izsv&Un=I^gwI^!vTn)k1q4bAKxHy`%w z<*b9FU^x}gk|ARCaDOiON*7FD9hc#T<$Z=EVe8E;rpQBn&c50HBp)vj$&M;Jle;&P zV7vkU=S*oA5vZB82CUw&zMM4){|D~xJ=tNyamWP%^+-0$ztolY{HXDjYwfrr5q2ZX zj;Zul&nG&Y3WS-ul%uDLsDcX>`E2BHMr3 zaJxl8{Ni6+&GJRD0Tu0ud9Z>kim@+Nzcp{)0ebWM%6DF3&GL+?l(+=u+n{|OR@vIp zbJNjUTJHz^57=YUi1?m}J-I~!R!4e|%|w1JV{%4s&m~hwC$6Th;==F6Ay0PIo$xS`72>s0KtM&i@z=q) z3jQyLi&y;Og0PV;f+gW8efh99S zkhI+xV#c{etyCz!vf8}FR|AO}Y@AZMt(T2z+jxaL{8*n8MSA23SjT#w!)K7H(}i)Hg_Jm3 zHGg6jRx~4%X_0lD+dX99qG9v4+!+BRH5;~C@u=ewgfEum0ygYuHq`v$Av~T!vD3O9 zucX(?eDY5`0L*|Ps;`+WMtc=zHYgv)YCB!7w2|=dVEP*ZF7`1lH*veq8m7bAFbCD8 zfa$UGYzhl#JV))St@O)pP6vMt$1W)tn^%KnN7JiH|Iqfl??xn7L>&2A4OPUxY}*vE zgW><0SFJ(={hO&+vFg=xQtu%V11A*%7twG1pzjPr_w)_xjV7aROaG?_ zO`Kh5Kc)v|2-0@TmFg=&ZK1I1oSN)QxE5N~ajXxba&Epy{1AS)9G4i7ewdQ0uEicD zgp8r?XbyP;M%;1K7-rn?)SK^7l}vy97Lk(@rY=Grv9FSfDyqC|MUXFkasEguh-;|~3{ zeX4{uDm8};1isNt=-Tyl!nDl_kNME3nqAoXPs3};&um?$KpixoZDNKs1+v~*S+`MD zSm=;o=IU!JFb3=IK7^;L&Njh?`2Gc0$J`qcbalOAORmOyh#YcF3tQtDe46OKCoV0i zMRQPz^f2YDY83xS0!bq0Hw|YmdF`=sFxNL3l;Fu-G9wzNEK9Tudv+Thz2n|9c|#cGVI(h&B+TJHb2 zi~hNwGQoo8KxdURVaBv~!XixhNNkRv@2m&uR3~ue64sQO;_#3b9yiinBLS9W9tpMKv`8u&Yf54cs#^Q)`M(&eH5_>fL@mk z*>-PwMhaF~fiau2kMn&?q&mF$>-=jDPhW1T{a1_?mfm-dx*dclucfp@bM5%>a{VBw z>ReqB@%^KDv#Damuy=HAx@3B=KMn|sr~1b~gpw9hC^nWIj$9PW@2Di$B9OkGX%($= zbB!Zh6TsK8x6g#u=M8!P>hdiNMhYXaZiA>^b&7#b|4hoLm9QXp-rKDH2+i2%U&5^3 z`I2AfWc$t8orid<^Yu~T^K(rb@GW{0iq<7AN7#*O`r1)Wv33|CQ9RC!`Tmb5$KFqg zdpR816E3q~dk5nw|7;8P`74E1r~32j%KPo$>hoanO1ta*)}`m`_5h>jBPFHhV>e>! z09eTwTZ1w7V|5+rmfof|ztMU|hj8@!?=R=bY(!yVz6!8r7+6jApT0()n)B@5| zj%5r%@4TXbg{T`Ba{1V>krnG%OeS`kIZhrE`PnvVwU7$$%Uk%kjA6yPhpAY>m}ToM zah0w4z~P?q4o{Yc6xgHBAsXwkUrW^lg2XkZ@sr1tni3!r&C8)9s_x!i4<9{S%HEq~ zRx4dpf2%;<0mdp-=ca=soocP6g5PbFZ|)u8AX;fJQ>B(~;ityasj}M7##ZMRG&!uP3==M%sFK3vqtAeW~q{`3rNa7ax@!9#VA|;h1IEY zRM)q}elS*#m4Kd?Y|_j8?3E`(Gp*mOJDRpXk2G2EQ-w-icxuXY=+8Aid2I$Td9w+H zge8>H!u|)&13aKW#;3^o=UT`Hjp|X~Bivr1pNJvFI?$g@*-FZe%d-R{s?ik3TYX@tu z`ySb@>1L9Tt3{ih=cg^77XaGFRZGv;6D`EY72?I}nNP>ko9BgffwD@$Nhb?QJ-7C6GPf@WY_I($5VnPx<4M_YznV z??fjW1n9l!(Hy$+G*_W8na73c2bme%xQsB#gB$0VAo^;UO|HRdPGME<%ThIFC{^3o zl|b-=g}ZMvM7E~UbMa#K?22G+^A&6sIzZQCNAkF)BHSc$qU>H0)EeTK%nLT zo&oV(+`zosoDL&U^ZpT)cQWh#h#z(FDOCly7Um?HxMFNBgmGDsqwHd^!z}3}u`sON zC!#Xq;q^7i(2jlT6Pm9sPpMewecR9WiLoni9`4Gc8C-4fLqMO>5 zB_1tGw^v`~6d)y|-v=@X1`<7IJW<{g6|@LD0tXrMuNi0-52u2z8YcT@LeR-XDgAs$ zx8_)cn3uwZr2vaj{R$SaL;sXyspf?ZD`o9d4Hf-9^hBJ?mNEG)zhviWWJ#nZI$Y;T z4yQ`yB`Uwt)9BQF?$~x{^SxR%4NspxfzATf@1X$5X0CV=FqK04Ax0{Q-)Ri08Cg&u z*r{aeEqW^-|Ib@|OPHcg;`YDDNe9VpWzqgC4*ruTZMXM&!Rj}+aC{ecEw4$nbmW+c zs#Kl_l`?c(yFm{4{V{h@##|{SDPfNKQKmD(-R5JtkEqqv?FTR)IMMm5g|a96WdjX( z<^ot{{l7o;t`-!b>iTYUr$_CJf9z|RbF*oQ;WFasIyMRsGYNF6BN{)5C=3>J5jZ5v z;OjEOhzrtc40}LnOe@Ud25mK0%|6!!{qK;&6O^g-rXAhxdor4O3j(GBM*Ha6I6=NuVGK719Y z=9>hf5kpGYjhaee`RLi!lU6>E!D0%cWF;qwaViBWv^v3=t=387?Ta+9#Ak6PMOI#@ zE_st%z!>&CCP0S#**n#n%SjsN@QVY*XFgU>#&~)Tqp&G};tj%^$7HJE3TfEP zMzZ2OkO<{5-D{xKG50;Hs0!*g6%zTO`w*AXUL9LlN;#u#Wj-`x;rh8FrRoI4 z-<C$s~D$n1xt~#P&|$GtXH2?Fc8Z{M)gqk^lG=bzMrU+a`h*7Mw6>!-78bZ;PFSl8gDS~xuqK@?43noKkw{o5&P37W#PcChR>9L=;=5qsk9l(@o5cJ& zPZoV-BLbdeQn*zl!Q|CgBNoFnT(WUX5n_1?814r#6FkB>&pVV+_IV+T5g6oGiQQh> zC@D>_h?U;%WGw7b;(B@#L-H;6%AjrF6qjb z35!hizjp@Z`t#;S_y8?WUbJuWkQ*24m>AwPs@>n2eC0Wvi%=y4VN;0mk!`|ke}0+! zpM;ym5tXcr!)&yp{E*0k`FPp}eO)npb(bDJ6{v9ze5@Q$HiEX&*LT({Ahz8&*eJV; zrm?Z)POPV{f`p^24N5}9yJSn!QiTz-vA{nupeKSute86JX8?F+bY}a@;(_Z44TiyNhs`equJO1mWtCdq-Ikt6^-%%|^>0x~Bp(DxT zACcztn=-23A=CW%Wlv)v*!p&ln1~xqQ@#Ptx6iG}6X2rE-t|vk+|PfDGL1OFajob- z)Fqepw-kWM=g5O3A<56Xe$+ad5Rd@CA>$l|%1Hz=b7wzP)^Dh7XWDHWS{9+ON8|tx zzz}CweO6t%BH00U^EpPsj33LEib%s8J7S=6OI9yA(y5L_^pIkJGPZRutiT#KVK@(+ ztu8c~(vX3%?v-wW9!^(VkdS4yP-qB&gX&wcr}nikSJc3IaDf44MX{FPA|cgtNYCF| zTh8=>_#BGzH<>{3TNKeh8E+Vu@zx^2h61wfbFka2v}{eS#o|R6yp`nkdyHKu ztCL|SrT2kVVA{jumhWBB9~cvpm*|V%T%G+@t|ucxf7z^^z%$WRQJ{GHKDN|&eP^Rd z2zrj}Cgew8YZZDhScFT7>?YkPJg|9=-3SIx%zff0$4aLs_aciHFQHT1*@@47F9$$D zncQTPpRQ`cVJ8L_-Q!92E`oU`xj$ku=qaMH6DpKrvIncoxY{W9GzleI;S%@YJ)O-D zc-y(@T6b`4_u6Xww5C|-bqH8<*ln{2?B)h&o!M`9T3jgXbi#{1LwgaL!Vb=CptH?00c& zz_v=cSpbmP6wbS-uZHP4iJ}JB%4K;@g6NUg0BqJMmP@@YHF*s#1x6B1AS-cE%o|-O zJnP65xW0|uI-9rD^w+4da+q&4aWv2O1J?_}UO&>XG7`&P@1$DC7E5*vcq8Yt`<7F`%rmaNVj;DxV=%sC&3=YJHf*1`M9PW^V=@WUKx$tTt`oXn_}?`IR(%!F`+k)s^us zb3q_?hUiLUC12jsRYbNRGK>3zAWLEMbrE~Rq#S@Zzn)!4XW9ij*M`OkGhcQ2XY#&3 zt6R&G2C~xWfoeSpn8a8ibvejWy_D68&YPF60_jMM*B|vfy3pzVt!1a7+PjGsn5>{l zQ3y9!)Le1AI{DF(byfoz^{CV6i^YvuF z-K18LBQ8aj#dh|YmBi&LrIp2>#}BVU3tSc@iz74j5SiIEH}|D&gw+`cA5q{-iXXsh zcpVbh*!QbGp%2mr>gPK?_S!Z=nc07G%bREC&sb|SX|Ll;a-d@}ZLRUFv6xiC>{T+7 z26sN?HTM2a6bNF;C1f(I8@0;kXyy_*G6S#gldcyEkKz-Mp`wx~x^cOZeO#C5>alo! zIM5smT&EfaF|=}nvZ=v&VSTRxC9MFHRyO*AsI8p5Nb@yXyjb*2H3ImiGXFMK`rpCL zs?<_-K6osc64+>61IVt@<-!3uSO4HqV8bWDfoTJeEG>EUdLb-|EI1U{mnTV}^cCG+ z=pZy;?h>*d(#OCxKEY)Bi5G_^*R5pf;f1A%4h#N>F+WzKJ!afBjYg z#XV31mEtUmCD0j*A1Dy#m7ts!Rd6e6%vQZqShWp6K|Nd(L5O;slT|4iG;lj<%2PJ< z!ZsIncC#x*TU`lOk1PJ)6+M~iabV6%4gP*V>2%Ej4t>5qYa6`#j%k5HM#FS4XJrKs zppExVy$(7|1H1XJ0}89_kRPA1?JUoNxp1F|HE^G?ufTSDC)=|i7GN8J{(t9nqInJf zyI%4Q_w zB`@(4|NIn=j9$%Ejzg?98kmIdc&M%9YJHR1g8kUssUXB5ZY+~4(km^aB%=t%AVnocN?95~Tj`k{|-cIzI{!b(4M$1&pM)G`IX=aWD5cT+z?1p(ndB{?{SO#8plvHc6%fv209OUz+CpNO%=gGoM=q<03fp zxHCv4L0;YN#rw~k40vRq@o-{Ec^}_TW^6`{S=ycvdH zi|CC{n@B71Sz8ZcDN8H9Ca2+o^cA$Iugfz2^O;M+bs&k_@S4GNz+mJf=v=^y=MMqr z&jg1?-$oe%G;8=`LT-BUvF^uh`{~TqweeN&=zZna;5R?OU!o*I=fDVlFe~$k;h;`@Xc4@s2MkNs^uxwc#32bb_G-Ak}_(C zg(I@>ka9noj6&Axv9&$M*6M9QJwDiBd_rM(p?s{Y5I=!aqj$J|omGe(8^q85cK(~s zK~O$2@=e^Gw=`--J-kdQi*VZAPJ$Ka+m?pt3tP-PYJFPHm3QP7B}%z56Kc1 zt}}c5#BO3L9B1};%^ctzRI?K9Vld597oX+#@!Tq>7awRFFc0xOgYNU3j(v*@>jkbLOXY2%o*+ zMnB)(=H{}W8Bcv`vj&@KD{?*&exUx4OT~1EYU|iqQYafCXz7^i0RbL#g>dNki!7?4 z5U_Hq5U{hl_*hgz0YPNPQ97YPrnU&xUK9UrKF1>9dU*z7hqe%|f9szBxMS{q9JfPo z{@A4MVahZWIU9QB(^1}rdDQ9X)#Ezv)`m_>3q89dpM9CTjjPTT4hu3F*PUT^&y*hJ z6&j$VG8agonF=Xm-|&rF@Til9~l7Vg(RE*r44mwet9m*Hl%bWsXhD3QgO)p~hYh z-hmBc=W^-LaT?nEpvdH+OYMdi)NTbx5>B!-QX=-Et|+XEb*aG<8)WJ*HoaIi`E^IN zy3|+~m6m)SxIL;j9tdy_wclyBY7_L5YB4l5in7M4Oyn`OkNKtUD zXE%EL9=EBd``U zpl`>>iW6$W@+D~+l5VycPu^tu?bVDB>aBNftr@L^2S>8XDhICN5>&$i&|8xO*RzRM zS!EUC-$LEvw7n)X(B#p#v({{2v%s@$XT?b+S#8A`GTwA?UE9`tfy1-h+*h%@2~o4T zy;#u*Y_9y*gw5#NUc~Nn4^XkWJ_UXe{~h48ySlZztbY$TTM4$vfVx`(@cUw{0L)PJ z8p9>u1e-4~^wZCv(k%fjfA-yc$vVJYA#}%@x#>&mtpI+MfM(cOBY=BmWi^92#`VEz zBgeXx#zMt~?Fe3m1fZEzFv!B~u+kF`&4SZJ49>_xWlgsE(JiU!;Ty{PYZE>t1e*1e zW>0eLFXJCwDpqC1|w5K zHDPJ3lny+dmmTH^JC?Crk2D^~;)X~q{dW{hLd^&>zhG{aYhUFi-2rTJ^0%2fF7~>y zv2sL=g)bI=?UkwpB|Z(LcK@N3->Dz733#6iH0j`Q0mI8A{vI0I;fDB%TeHvelva&C zQ3qZn5tr;r_hPX4S0iya+d`K*+BGR%C(x&-+}I^IN3yCmi(>2Jr!uFw13iVW?vTzG zcw<#>VoS>&1gw_{9kw}hUYw|P*x@TxJ&%cKksTcdEiQwPg~L{=nI{X-sa{IGIQBR4;~MtTHK%4Of8~q+-!rs`~p55;Z4eFAuFc9k2wP zhyw1Ue2<6CAyS_2RG8M&fx~`BJmk|GM*e&F3D}TRfKBIKHxE+!%Nf@Cn*G7Qq%@br zW;bTY=uh#LY|lyRYyWr31H?`LFcaz~M9Eo@J?0d@c;b3^nzWFONJlMwExn$n(@9unZIIWLB(aQhS zI{>Z1&+7V=`TsK(!2fLKKXa*>uK0h(BC}8d934VDR#aWhP-M$w|N2vK06X}`xC%?Yz5x>`ko7HNjlZ!d7WxLWF8 z!ymxZ((nUvsZiv6xmbm79?aQrNe<1rfrn3;2TVHg>lFIG~TM@4!P69&Yh-1Hll^Sif!FlrM`jMv*(&-W^D-L8&0r{OeInqG% zc(EhH2qBIgg?imK%Ue7MS*5xB(es?x7%uR?aK>6ud?8C)P;4%ffi9r14)g}I0mdT+ zx7vKCD?*(lj71DDujiHfzYS=n^pQpzL<^C|6S+`elWEyJxI9o_z+_4Yz34W5wgZ}Sr8f= z%-)R5RYVpO8WX*;>P_{`c2kwx29sU9`gPL_* zL!D(fjB|b}jlWH$Wg@2=HW<6ucL?1Xr_Hg)P>E%c{;n_^9CBd|nH)3Q^yHyUJe)VM zFA^0}P>i1}%3swNlb+O3^I$2|%mHMahR9Ly-dE z|7!~shQGiI4w`UA@Pt*}S_9WcFPV~}86c!CCg!(Rwl~&H7eG9YnWmar>BG#A_L*1< zKAr+I7iz_Y=3Hn(S5nskRuY>3F%nRl37xH&S+L*<@c))UyK*xXXii9|I?u9%Rqh>F zj(280oM~JTvXtBHySik?G^x+2eLNZPM9)6&_;OL`h0u=ris^(O5J^#G)JgJn^vwHp zkyDVs`|H=&!@;r7`^8rDm&1>*n;KFdo|lr3;hwBa9N?*pC!H-VozJM8_q!T1dY@jO zkGtoZ^IIFgtjx^!7glPN3QH83MbUl2_xxp{1K~S7mp!5@U7JaXClYvFY2I5mdQwLk z7$z?bSsT*Wz%^Ko@Bz*^$bjj>9U3q*pX5TT>S5;HLo-S{04y+Xw1(t>OcR;@NW5-k zy*pY}23kd#&`;OY;TWVmyZ>X72bieQDxO?~HBU0%fxYBf{>&w)gNBnGr-a!{$4GNz zrZ2TR99p*`d4R$*2#P=+mpXGKEBsvTDR^bqM!Xd1Mf9P4mQtgXa4azAQk)4dDsq6z zPpc6AZL9~`SwHpC*j2 z1s_13U^#_Sh5$n6xxR0OhZ4t0*Gz|T5$=D{`avIF`%0psig2>zO0W5#e_!qR`jV0v z?#IQnF-w%t5dI1zZyE&F*$+KFpv~3wK~#0DXtI~T{y;>uTXr()As4*14WnTG7;o6^ z5X4SV9q+g}v*?eN1j(IMz!6)k#gCMH%~?98?>Qouh6ri)fG+@Ki2)1jk1+W!b2C-f zobCDm@QtT~P#bA4i&wa4e_!1m`I8vj4t1p4haNQC`n?_@h+2aue(QAXA zMVMEe88y7u5Rudt$;HoV-js@(O5dO4gm*F<5xH7O!1Vl!X|z5p@a68-!0p|#Y)w*~ zlF}mMF{v27#Po_j6l#nnAxXyPxO6y$#Zmf;U3{47${Mo=W^kJ%fo6sPzCYGZNUTsY zBx8LL9)&wC#c4u=EYI* zYE#23o7di-P^F}NCA89|ynSMGOV47hZi~B|!1?p+4-&w$r1LTDs$Y%aFYkEk^vUjb z!n{+VP@O``2ChKfeUR?hOYd0SJhGXuS*M++tEZKYX#)wLKilwu&+O`55S(p*EeEm&$ z!Gqs-aSbM|*P8!JZ>^@duSrcSRK7Bd^+FmMtK3fTuOxx!9c7v4gE)vm8Y}W|z(zQ` z;4Jcm=TPJRSZ?K%pap}vX3eMOGG}UE-meYT73aaK01#({okR9gS9N_!*6T-!@p z=#Q>+anldG041>ZELXm^1?-v*vvxRis%bY%8dk@sYRUXC%g5$Vy<|TjA*M9wGW8Z; zM`cBox5xG91sW<)3vEjo93rSb^F#gSO<|L1@i|U)-6nt5Sl_AF zsI>=gsfu}M*cmN!byliV zig=SXeLOrKz7T#TVSZarqiuJ|wbS)3Aen{Fu>E_ePj4xxb=C$4N;O`F1ox*nfV)7{ zE06o3o%Al*;}8oCO7XGtNqh++JZH(}c*l%vKIl-5v?$nUX&tA@0M?tslklV&Hm;t`}Q5up?pUXy=+;?_b=7u`Xva!l}w#(d}=tzI4& zT6VAee$)DW9V;T+lKDgrfE^SVq#xwU=Nruvs`zUvYyaK@f0V?uxe#0SBnl}~0F8$& zJ;?HLQAeuxHdy0c%5ECjc;YW?mED>yN98ZRBtlMNC<&vu4DM|ya;7?6%vzJ;r;SDI z+jyL_mn@L4Y}stxB#87HZ;TvY7XhY}=uUo_gHTh{nS}mKl+AO}B22Q?&dQVR^&ucrSZaC+=VB$HhOhn9PDU_#mc~pGcf>1SI49Fe z1Gogo3f|_)(ZjPdY#x|GD=Z-wJ${kY*%hDp&W&1r!tjnru3$)e%q%Wfg3XDkT@PdM z*R2MhoEP4fJkw^9#a4dJ(rHUC^CM0-ajz(m8futZv*0hH+zQ>Ayw9Q2zm0?JH0WH$ zb?9n-GFZceFJEpyaHC%+QdmDAT^Y4<4wH^1kaE_7PJ~w!=r9yBqoxPr$_62x-IIrW zjf57@7Yv7~Q@K45N{MmBcHzuM8}3*2LA?L;9Fgdd^4#qJICsGPbCE9=7#a6%m;e@Y0rP8`0HU-a#8eX7l#jW_xt6QU-A2l`>;ys`d z|9!2A^#nZmQFGv{-BPQJY>&39u4M~=cv06DvtWdM#jUHH!;KLQ;Niqo@`15)v4JCt; z?aqT}EFS}a-R=8M_yk@HiDR4TpvTz>M$2UXUmh2QO- zG|$2aBT@oq)$Vgjy#_Ml;4S;ix18sl7Cc&)$92Mw;)htgpc1_-Qs~1b9l@EbBXRIi zGZ75YcM!+W)phl`nAtIZmy=?8)e6?HHZ=BY3a1yi z5_jAWYfHtlIUI?v9WVuT^Z}cvC~EckaljaEV5(f0jrfHti+UO4M0v5!qts!O(j}b{6gB%4w)i9yB5Q8R{ioLR!;>>) zr{v=nj*pM`-CoGuQwk~O_&t4rnFE~NF51GFee-sH?C2bMb;gqPTH#^*Q_GD+I!&12 zi~paD7BKpKQ!(Wt=~zbccwr6lcJXMZu$N49H7}^OM{P*MZ~m$9v7_e3Z1u8Z=C z8`n}Gw3)p5o}oQ5B7zZ%k`$aHnqVw{uaWF>#p5rQ)8GvDZw!+fg<|dFH0K3`B**mv zj<_eg^~SN6)VCX6>wGu#VI1VZrLkqGFvUi-cLActfIHo{Gm7Dyt zKaS)E*4CsBIAAwI6-J7gB1u$P%omC47~uopD+yO-_t&C#nBr4LigqZu6MubAxQr@< zx;w>-@>SUe6(d0;s0hkcd(8^H=)9Rcs?PV99nI9xa}G zwB~<@hcym*MOL{x@)xKor`BMW90;)e1Loi zxLZg{fxvl)-`EXnBDWy$mNJYpB2cLpf}Fc|R{tQ~4$F$qS&xAlxm_;c^>zS~tZ}>o zWIngJGD)udmc;zan9=CpK=%IfMwL3_3FnI@y#xTO5$>hMo}R9KFrwU?_Bg|k@B^!W z`+qv}&>wPloHC@>u?Ky5R=Dh?mlv<3)!QkOqDk;q8?q-#aa1WXxz@R&1;Va`sYsqP zar(4-k9xDlXLl$?S>oIu6rt_CzmECSa{q+yC$e3J?`N=73Tg2$!L80*D(|~sHm*%Z zU=5WMjsSJhG&@EqWszH@2)!ii%StC{b`eZ697<@3|fv`@Xa# zGRZ_?Pkm1$F%AWw_Rv}h)+ltwCU2$?NNG^Xoj49fjpz?AOl?qZCA56({&PX6Wm=RE zDqCTh#U?y~p&6|A0GanKc4`Eqkdgyrm1^5|uN^sPuAOrz*x9et@E0RK|MchKwsAxUkm|Ittq|muxz4Hv-Xr$_B;*6U0dCf@$Yp;BH9bWQNFR4KAs!*G`m~%@wq$330 z8Sn=<~knT66xvaOBZBw*q_2 zK6QkEuC_Tsb{swb<3My@ZK}m3`!K#NN7gL3#Mjo>+t;kbB$Jq+*hvFH1Sou2Bv*Z! z+{~mu$^MFQ_xsy;O8R0LQ+3SM2Fn;i&(0slhhM}IEb~?wa84wJuW=-qW4|%8)*e10 zVoz~8>PExjC(I!FM4u$NPY3sZt{gt3+RmLyu9T9j{I;8%K0#I1(Xq~FL8dGh=><`_ z?aybm5pGSN>0Q`Z&RAPeP3(ESQCVJ$si6?$S}gf~)BJm3cDtABp;3{&jlNgyZw^NJsy}TLOfg5(>W(dnC>4B^VpnVn_1vxuV zo23yhbD_>tqmbHSIM8tW1H2ZB7fG+`N2?ThwgS|w^=}(dSxG70N#ld&isDb7Anq0x}zb-9rT(>nferxEUiQiWz z;wq=lQXy}7g-{$rv77gr<6#@4($FGTq4ZK>!t)A5I+y@Y@b~wXN!P%t_g>RP)+X+) z6a|U=(cNFjb7(pCaI`cQ6i?qe$vU1BWcXsLew~`7SJNW`qiT+X>OC8f?iM4&kza68 zaD;75A&T;e&ej(NtHQErE~ZKme)6O6)YJMJ^}ARqfrK2Q0#fmb_^obn?1rjo)x*V# z4Vic7+LEx*S#>q8h!SUL`qCd^510q4ZFD$Um-Y&d$>owo#z+zrLy8cbKxXhDuRzhH z;)U#24Mf2wmY7X&fikDK3Ys?;I;zBAyMWX-OJ_(XN?Y~!mNcA_R~?5qd*WM#enqBD z>ct%@)?u{98M)i9x~HcttG%Uey3dc7Hx=C?6KX(JAyG*UtBgtqS2N?o1<_3ucbZdn z_NGE;AkhL*FFqgV>B-AhPEZ#pVxb{&BKh!1DDT8TeCTpi)W_@Udo**!q}z$=6UW@D=z>pr`L0GNdy!9LFq5d z+L=qSI$3ULPTU~ZJG-@7ZXJd?hJ)cftjXIH2cm`)PI(x@WrVmQ>SpY0kp~+1GKE8| zL=0_U=3;1^+WRn`_A6l8%N-Z|{dwHyN^RnjY`{oufMM8%S33)QA`WI4yR9|h=S0SC z%!JmLsZY(2gNP)bCfTDe_oq_Ov? z9&8rfOfA#Zk$$scQ2fF?G)2GFK#wr{D9OmEut~EP<0^*J30$>g&}ZO%IKVZc+nO{svJl+-y zfl}^aZ$1#5DQww3XM&B$oRufp$68jfP*;N)%otNqgvf(~C>fj zUU1y6f_%c^kcAZ83+ZQtt;wTSjU=uP&Otp_%-8FgPA4b^s@@P3;eUQ?%KRWG2!;1mcl-c0=pnf+X$5J@A z#Z*h5r7EY=D7aMfI4Q5c(`P8h?X;|sHqlvM@}c%X?fxvKEY~b~^sk%^D-eZ(Q;7$X z?<%lpVu@P{9OR;*R8>oMz^ajWW><}d8?S*I%iCKdnpqZG)Gtr?I}u(@7pnFLmI?=c zXzFeo#?w=jfBU0v9p6-$&!?BA)DSGO*K7^}q3S#*g94W2IYX(u=>QIX#NlPTl2Or= z_w8I+2k3|Z0j2gE8F0)8vZenRa z+Qm;^AS%@TYNRe!=v%M|ajOVOlTwiw#g#B=_f1q2u>yn1njR|qH9VOf%%T`vk^ip` z<1>-x@?wc{PMJycpWl|7JObKoecidi$%ZM3j7?RQU!fC`g<^jUaAh+J8=`4xJ~vs} zrba#{ol$YONZ0vIzNfN{uy`^Cr~ZZE!=*LqAyqt!pQbF8B$N+Dh}CN|$EbUqY3_Bi z1bk$vn81cUEE&HRDX62C7}1O)RZusjJ3j4{eYjeH7uOy$ZU*8gKgb>)?=bv1{N8+q zpqDswM`oL-VAYm}YbH1{L<6Zz#Xt&eTa8z$%@pi@SC|y`o`+e(B=-c1ut-87&ay#N z>U|}qiJMQjSS_@$W#91}t3yE&`D6TcGk42G!Nrtb+wV%D(Rk)Z?i_oxbiOem+j5tg=O8TCnj&DiBN{WcW+Ea_b3;w( zj5&Pb7SFNBhIaZx4D`pah3ahik!_7rP>pymV^mgG>B@#yKJqgWXgx=Q$@bNJSVTnu^Hb9u$Kkp8CcphE!*?r z3&J%t(O>it(b*O>Q)~kHsfKmhc|e&!D~edhX^$WJYdThaoUt1^!xupti!**dDrXEW zbR!lx`>Hnhr^GiEgQDvrmzi~?+!MFLGO!aozQpWi?O@RMr+fKNqTvv{kY@)&qUiAja>&md%ylY&32|4ras35Ep$A z={t`!(3NtKvGt=w8#MKkF%3MExePQm-dLMwJr}SpK99Lnofq)NT+DsloD1BI&tWbR zCOunHenhtkny0W({lP^tj*Dc>iu?O>lmND)nLQIJ7fXm`pOBJe7&BlA{1%lPEUfIh zbOclTgc48{2KrUD`BLyV??2qQ-K2Y^Tu{>%b0W- z=YeVaUj}y4cE)VB619TQ#v>CtPr#-&ikN=tk&wV2qeAH%t1p#GJB`sM{39xCfM3ul z^g7s%ja%=?2&X)!U-V3n}e-ebO~rQQ>euDgz-Q8hMJ$C|`lbPu=DKvsF^Xl`?% zT~lxu1$F5IR5HbExV@imw=+qWE_u=UI?x}MvwV&wZWSK6mVw6T-$S$|Aw%>Ox6`Es zO)(1sxJ?^F=&~?%4h{g?ln!FsCL{$^VemxsmGJ7`f-fRI^rl+IDdVwe=P}u5J7u0d zw6tR|lXL5STFLt@zFt*VbTd3s}3W95=r-^sNud7`)bPVDa@ zTqnn;S+Sxg~5XVNSzrQ>%T(gfI zz)k_V(yXNUp3u}q!`!p^O3K@wMvu3EDB9)}fq(W&%h9}PbBFp1lQvX;EBWBLsy=b@XHv| z=`V%hefk5>Mlbb%f8Vxi7`?mh#`E`+me1cbFrwd|Z1??d$q9L!_n$mfhzA+XFB6{% zgnH$}v=v8FSKAsSQ?CdBD_$T{Zu5r^G{^-Y9T~C67vq|z0Uy&>ww;SK<_=E_copbdT)Em|2k650lhbT4(PoH^xgw{ z?*YB{fZp5u%>ljlS?er4w@%5Ftjgpe0WEIVKDtK2lmq*jcPwCxLe90CL@lGs!UbY~ znNvZRZ0d(Qt)uT=KBP0Z(rGYm2-GASaoymETU+TDV3UmUF2i^{-Jd~AkRnq1;2=uC z&w3JOvH?SIU*XG>PF_eg+;PAnfMIhL3T`p%PY{|Ft8NC^9Y}J%7BuIHBbn38rBt_= zW8Y|S1ZG%yUTHed*o<6il2Cz2enC=oOdS2Ne3Xyjs*{o}SjmK;e<^Hqql*|cd(`-K z=l#C743!9PMJ-d;z`{=OZ%nI@>1+$_5!sdYijNIr^YQHAKB81|X}Z645N4i<&^yXA zv&p?(_#mNZl_XkyvYc$}g|}|=hUYz5NyP+00(cJUU453^Qe-~;PMk{{?KmOqj+JY@ z0^>(1(YOSSnBH@m1}i;5AOa&H_I3a#4#%bpf0b5FbzaGFc3_m8qdjI6<+PaIvUSJV z?HS!e{+*mY1igzF>cbt@iPTq7tfkjB5pS7@wo4RLYgURk@BZ{n#mbIeJmE{Hp za{VgF)m~E@1Z!2PE`oK8yT`QQd*F)5oNH|pqG>=V78l9^!M7NIm>gd?t za$waIb=o={hN%;{K~~q3cYKQTaoyb1b5R=A-Br!a!A-38IayT4jKw`dw z{jS%g3x6we72Pt~Zl=)9_C@Pq04iCrw92|NrkY9X;qQY92h;OSJ}ejX3`)z4A(6yr z@Hh3ZA$O@66=`jV^>4N>eW$7v%TI2swGhO_dRU-jYE8|$fCU=A0q4}3n7!0^&_PG( zFrR>-x8y~}$nS62dV+aAm^=6fQhAUJXa`_k3e%A+$?x62{_qgx#b{~RQ?x?aqb6J+ z2s{q}o|#<@ix00QPGv?eAg?KBR4A*-(JX8HxF^!@2bVf+NyFtz2KXiKr6cYMxdMHp z{sgQ7@8t!ip>%Z>b>ZH!4z!nHtN3TOc1jj5$1PjeP}w!z2PHnbm(vLQ-D`P?PP1F= z*(zb4)SMCg;xkuT{Xfnklh2*!0=sp43q8AC5GnlGH-xA)b^{t~p+C3@maHh5g5+^( zhI*td?4A@*+rj!@@ow#G(jmiP!$^g=Y_Jp;KUOrGHLSh(QTMUJ2F%X@*z&EKSd<0hprz(XmC=%zlf0B=XEsk<+NP)6UMu{t zeY|Nm9B!yyd7qw9_W|_1d5n(HVsELuvnRQ5jLdV6Nhp~$Eq+TOOO~Dlw}U^^GxE*a z35Z=%A0c%zIFx^}W3>m)?q1C*7P4e_tR%;&gm>5-PxQ%*{J*S}y?MUg!DSDxFRfE2 ze!U{c&}8V`I&p3_n@ZQw$GCW2y-~bFi0G5^vDAx+ZkHCd*c;t2-58eYnJ8EG95*tj zJvf2mxt^B(4r%qIWJ@@Oc(xvS1|vo7hYU(s&*hy_pkk!BE1B9?@NNLbi;FFgg+U(X zBSpDS^$;7;{LnebPlmgFawVPgssm;*!ZsTtVnNqeJ(0~rC>6qUMqY4YaY0wBJDI| zHS1{gwp&jVTP4YK;BX{2K<}L)t{zrrN9`eBh;6F_F2${5@ib@-!&3MW3h$NXan&+LiObfaffeSUa}P>KI4s zZJIF=%hFQ4EBOqgeppMR5RZl_6F40z)*%eN=pJ#LLQL<~MMjn8$v2ryZm+fMu2(*U zL2PTgu`Ys~{%~zL|7{($Ezpr%JolNA@tX^6rX^6lGV_YCy9DlJR8 zWG^i({*2x$hO?fVS(~rB#IM-!y@k5zEy^-HjtpCMmbTjTgB4pEb(3*t*0S=j7;w`1 zBgt-f@rGOkg*~@*wN1=~?L?a$;3f51H1WCkO!FdR>J53pKI>n-n2;Br7s~uk=mpdp z&b_yviv=r7B#t?=FAT(ed}@jzL*fs?chG0$Av;ixZ`ct}@PQbEY%UirpreVu!vh26 z+*MGIgvh8#+t1nY__fns+3Xy|Aub#Qx(yR*YPp`#elmDz3B_(oZcWg=h;@6DoZq%Y*?1ABt~A*ehsTqJXHO&rM{nIN?}IjOGY=>f z=@X%Ykh1F)HT<3ARi=3nk>~@YXlPfNyU-D(QL=$ql5>V8+w=`lTr7vQxV^!55QLjr zW=}9Cw-4Z*1ZV6R?cTz@>T+#(#x;mWC$m`K3WBN2X30ilmhtkTJ&E9!*_SZ6;X5P8 z=DU~Rmphr&o*PIdtL4oouKe+l@@RCCg($JbT{yiz`u^kPD5A~Vo?5}1qK$cDt7rpX zSnSx&MYHJvPbJ;#UD-+h73dS{Y?n+AmXYe$oJ8SsMe(a{gQ$C~1yosLy0$F$)UsSG zgCJ7!XVwjjIDpFm(;Plu_m-P3-U0A2={dUth8M>#>c#K=nH%$Cc}BvD)gnHq)cpMI zb<=2m#)=hrd%ej8I@is@>eCX7jzfAv#urTU&Pm1ch+GP;WgiCba30~>HMH{4?I?K{ zd`d|ovy36wmR#WSxYu^k(wQ}e&2E1Odb-ufU*gui{CIJC{_bLyr$1ZC`>B@GO0g;L zJD*XIzk@(nV}VW|0d*P!cG?z7!Rg5rse#45SIpz(S|?Qh)gYywkdoRe%NF!!QDqtK zuPzyzgr?7zUqE!*@wRI$&a)QHp6+-Iy=42OF0#W0)h#!-=(_Pan!gd#_m-_m3HkrB zggRYKl4Lr8Re4z{R?e{bXSU)ompqL7BkO_L+jYG^vc{gMZ-pkobO3)Z@xmxL*&)Tn`O)b~7RjIAkLh*jE~axHY`TClH*HR){i6liPb* zrjWE%P|d9>-&@fLpP&e#OH`PVOQAGHEEyYz1x6$Uq7z}&GGwo`*Tn8R%Z<{Nu+m7} z*_zy!vODjgeg?V)E948!R;%j}2{UMz#!&UTN=JzE(eGQyjsxNb8LoK|lHuVJgqd;QkDP zPsZ4A@~m`;XKguqf<1`W81^l|*7uFA!Ko#oPCiAp>P&QIDGOTCoM~2Cn9&+@JH}h_ zpa`VZY>-t(zOp}6@Iadm> ziliWk%{KyJ&j@z5xdDQrATrU^La2~A)Ax)C^6J&AS0Jmu`t5IjO8{;xWeLwWe+a+& z_3KWoXng+T^xbz5?q6tyrQxw>(?FwfH6j1|`osGgMYs7a7>WGy^!FQDW?%j0SHFsQ z!`6U?G|xKir*LU2LTWg_KKqHo9bQQ>;&4HuEq!A`7)@5> zmaV52uAg$BYXUKeD0MWYmL$T}apu{T%a$xk`FA7|4@BYgVP#b`a=((w;@hJ3*W1r6 z!z$QD2xY=UH;!w)P+^y?tyMc?}tE&VfRTs>DX?m|_~y1MqnmFm+f^ zbsv~kXY(xWZ{05Wh3>|(Ay)fwxp+)hc55)FwGtUq%4Tlnw7g}h=f5*Zi&RlCJp^vp z48d%`zCqUwObauJquJ~T8Hg+~{T5HV`MmGMMYBDcHtKs;R1_*c|@Nd%gpfz(YcaYrLlvdO@#LW43HtFZqy~vZY;P%;C4nvAK%g- zb{JbH*r<`WygR7!K>scwK&uW#5aU@p(eC zEJDj_iWTk-LiNN!yCA_q#W}Y+Jg9qNL2Dzuv#o_yB$o)cA)pW!{s?f{-3tLO5+UNBWJ?iGB&MZV_gDLeg6g!>%1>MEzd0w+k z)MnZl`hkKKBZx_R)@wIH5cuhEl|gtO*+B|zMyRt}*5Ig4uDxcNkv~Z@q#@R=BGWfm z!WL@!VR@nn$>^MAYGRWvK%^~bL>MwY5Hi41IM4oS??pDU0~)k~&|ASju~J1~1J+Ly z=@j2~nw3gcET{0#w`M^*P-M3~ToEIEYK8!Utc3E=lSb(Gi@O*MKKBfCX%k0TeeluE zr&(sD;aD?N1(#%omDmQXMl+-Ycn7Zb9;YAg#v&j9E(Y!L0A&oH-BmJt_8p?IYiPp6 zI}nzWu#HlWCWE_9k7ZVOHs)(w3nUxfm!WI(&P7 zP+Btc>ln8RKF_jFee>D5YJKx@i?!?ds_(eEa)Fcd_joLZis)`|&rw1j5;gY~#=}L8 z2E2FYd^D&5*IqMIOtZX@B`wz^wN}J+Cr;X4>tnzWQ{KgJqruo%%By^IC^l#{T)}&& zT45_Ux)fu?f?>kIAQ512w!NE}uYW8-`L-~K3zEqs z2&oGNJ0!ph#{*)W!G;V?uP7OV`8rb=j>HqFp#_RO&`C7ynMQN^ndenbDrH6rwdsR4 z>Ve+7GGG9m#|6C9Fkxy^Wtt^7o0lU)$5~O8+UJ51*aMLN?qioKg-tr-`UNL zH-^~@J6R_dv5z;3Tw#|;Zji<|HUDu&Gv}q9vjql<;X7bxP~C}xXa>xfhCD>(%aCF8 zS}yN;i>Q9@%JcIc#!OEdpaKdl0)nv=b0D*N<cB8L35t?Dn+HdFGTK zJ2{FQHFtAE~XU9bPe#-HZZyo;^YdbVDo*JtK40fmS}n=d?0p74V+rE<)lveVu;N7pNq0AnwDpupUtI6($o!Jay1;yy zjLqC%zA1N)X}QF#AeS-NAF&|Q8BQhiQG{*(?e}Jq=G#9^e;<^o`@`SNwHXnm`?uCx zVG0S{Pv)<`UCH#iV95-ZP5mv(szC@7x?Io5m4w(qp5lpvv6t-c&gBK$xkk4d^;%9msjn!S zDT)-DwlV3EhjSxE#H@o9NiEaCW~&)_529MuPt+Lnh87u{l`L4ol%nMt1NFIf=@*q^ ziz)*Nwtq3Gz(UL=OQcY`tP&f z$)CM4)V~PQ?+;xV^@3~IsJRiBoaeNEpdWpLgLgxk<>c9zIU+gJG^Lux<%tY;G`^$0 zWU`ITf`txZwyYH5)FN5%5Y^@C(^*tMpa;R)0)C`6G;3Y^u-BMlxIQ_@UZ*eX|}C!Gn+OBSH{M0I8Nu@ZQmnmdYHzGPa} z!OD1@Mz*f4=!GK$0nhESwH|GqP*mGUCra9D(+CPNn>^IbjCIA|$u!g^lCM*|jFMA=gU9C2!x1ZGma!6+&Ki4K0i{r25M9IlC}Hk# z=0mR$3U=>>TdEMU+=cLf`O`%PZ|gVyNdMrBjHhCai*_Cm37YiC(hSLHi-OKGBeevC zo$4|G``bO1DjH2S11MX)hFEl4M;hDr z@bD%GJcp3T_&0WECj_l*(4SMFm+*&^J$(GeoS8#fBO1mpKfx^s``Hr0TAzYHLj^-#ORfVCAk^@^A2w4kM4qeW}d7@1En9};G?OT<%5q1W(fY%wg6 zbAVdU`=DSJ_q2C)-?NU*dUsD_)?u|kXIAG$DLpE^;ZR;4qFJM(vWbQvA%u)}wI+ zxKS{gpo^Nxkz)~+Xoe3kNNn>2(;Xi0#?0eEFG($3gjSh=aXaybi`1MbxHiFFqpBof z%v(n-kBS%k8yA9Is+{U%#S~EzAk>wnS;kWDp2LllAn{q(U37nez!<+Yfvw6KhKr0^ z``n@u35Eo7y$(y=H-vXEwkR+|#*3K?^u`2+6Hh#FldkdZG@F`Uy(0QvdOKO|leBOtlV+y_E_U0xVa;ZL1AaO&XY`(B znfZSgSB*~M+B!$R6o{R{m9vD}M6u(vltp3vA^FMfbA-T~@{TEp>2gVGW~-J=u##$U z$R= z-Y*^UlQYx!+{S&ISL$rzAc{LoOQ?VFSWo5Uf_!9&ykjMj$i?S^3ZtUGr93kP;s~Yo zYCuKbW-LMA67#3tZgk#@&>TJnACqS7p2W0sK&y8;KEvj$k{%bW_jWK%zcX`OrZ za_|i}$ktChxWnPlZGR4jZpbh`T54(^J1Z-69lKrBP%AUoKDipbar_^>I7mh3HuD*6CBu2 zg7*x51;HfZ5l|Y0>|VBDlyuHA@?*uyHIXru3*6)LlAvt2;wk3gIDgnatz_T~yZCH` zaKKR8S+j08RO53FqxXefM!&S7n2n*JI|<&zMtz>XKOe>p2F4naM>0D?PCHfrSJi(5 ze8xw6dP6+9W$TFz!6t?#sQdB2poUv{bX@ zfT)i8$yvDsH1H(4?qdDyaFO8i2=d$|kiC9J2x)iEP^&0DafKY4Z#b$O^x95Z?yieF z(z1fY^&O;JJ*}R-BRxKbUfWmsX}60?beA}iD^X@#Az+8f4Hd1LMmI_u1(~FvYBjZnadHyY-vyV=Dc{I z>C+a39y-JVm@gr>Pa=9eF5nZDH;?3mL&4PbBaEvFTz^K@(P31Umd$ZOK0RWOMI(z7 zq?1b25j&2nZx0RxV6(Zbge55li8juHm>D|o0pGnP$ROE`FU&kGWWbPlHiDoah@iAJ z_}1G`EJ-E9oq*mZxc=Lh#!e!hX=8jVC07hjhSF5C307`hWI=f!Msk_*g^jzr4t%E* zM31oG49E0I&mr*d$&lM zX2G|7UNUmFqC&7N0PyZJLmT~0YnrU^exStgxQv;5c_o!5IZakvc)U^JpxHYct-)`` z00`S_@oh`^Vb$tJP43=yJRtE#8+9d1%XDBtQN4s~5PsX=!^2g<$z|?co?HaCJhLwt z2_lerMk~afRlNnan%MGLCCpOS^@ZM^5lzb_)21yVs(jAMK9KqSl*HS6vD?PHd#cou zkM1qV;Q>)=A!EcGj16`5b+JgIxrRR3JF*h}!cTk=T->`Ket18unBYuoaFVOlY*Z_D&ZzYfsGL6P_}m4bvzU zIfgg-0X;4_;tfLkP>U#^b5Y}7)_ZSm6LapJH$gFtSY`m!Tsg2_r{y}55+ z<@Uyj1R48`M|dny=eotF1OeG(HF9o4gy_p4t5b2)mtLIku!8I}ODvj7kyT6H3Go$O ze;DjLgU&{c>Dsm#+HbHtLBHtWVCmz~L8GbeAm}_O=!B^BLH8*um`I@(;oF+5aeYSH z7F#>-dDVG8auxj=*JxlYSjj9RTbzB?B{hsXH!Rt=xgb=md#kiYt6WbjVx|8YB9>xF z$#g^#_TXw9uHojgH9p<|GR1FQu2WqUIifeAxaM~a9BTmaph|g75v~P$a8(0$qo0Jq zHp~}9$kr&sj4mH1(fCPk>AW==uH0+?0*3fWor{BTHE zS8T?jJt}X=iNGzG8(n1Q)M&3$Jvfi2~s+r&XGM0S>&8!};~A+Bsq?n@aPc{)) z$JzZ$o5MIYgP+=L`x|G&3VYTBqqAiFZz49<~h%3nIYnw zhv4(=nA^~G2*Ho9UXWv1LeB{mBx4Sp&T@n`O28431W!zY`*9Q%={^J+Y&E!V8?|@N z>pO=Fa}mwR_x75w_cBt0Srqq+=e8Ts3}NqPqL#sD6oU@gbm=|Q9q2bY#c5PXv5yw! zxcN1;!q!&YC@6UB~?a^a)wTe?)^HyxcYc;c6xJh{)T*Od*Pwn zKO;9`*{~pK*;)$xI41UP=&}6yBC=57g9>sfxIXiZVjf~9g^dI=En8>Hl4}c%jvbk= zU2C0P*0n^SYPO%jD4rTuJ$iIq&{zI;sAA|su0M~vdf&NXK(4HgEU{`R?%&aDb~J-a z|7dpPM8V7l8{2!d?sP~^wIYr&;iD{71!)Tmh0lahj1`NBsf*Xlo;dYQb^q$6@tpW#Y<5fOW1`OusJ$Jv!wtg8kVtjsy4^*xpBz1LkCOYHXN zx?4D>x(k**inANmdew8Jh7gErugTYol1+UbsRhplPtE7t=ksW zTBP*`Zp6>-E5hmPrdop_U1A-m(qi)#CBNetTSE0`E|#i>9kfoxJ_nDh6y0?0yaN}Z zhw{Ng_5Q;RRA2{h&FXP3TMU}(F*-^!BEQu75(^a&{{DBc34DeUEKPuv3@>dwL3)EX^B94VRI znbtKFN(KeIDy(}O^FD0_psOm&u4KlOwXbi0_LtObNN1l@a+=-KwR#hdno#(RP_0>B zAVHd`wjr2n6AYHcb;ZROlxLosRQ7`&?mQ*$*&QpJ;}GT}XhJpnh9w8-s8XQnU`d6I zE$SH~U$mOV0wk6i`UZTtcrW!8{>c?Mwzrg7U= zugZT4GXjUCbmzkCvZqPhcMGx;9v19AlgVu;IY7B8<;t_e>7{Ez)08N_6i~WYXp&LQ zM6#X{i`A@xdd|A!KA>AbnXL&0q0JY6xcaaiY_CVL%U84uDZn) za16+S2DqS_(&d8Ixt^ZR%1UUSvlgx+0i*4PTi$eKK`$Xq!RB_lDl9y;&?V9Mm-56`g%?xynI)Bm^8xO5axA3ztjC>r zUCmKsgXUX9b)~g*uq(YQrK-7Uu&3Xfl`GEz?J1U2CD&`i#Moy&TPUc;aR#^x_HtTL zMTE4Wi+C7nLN&xQhIPI0m*Logot?$PavnoIHI1qnxspn8vwa>HCnDzbnOwi-`N}R} z0xHY%zJmM4ABy})38t4ffP1Nlrps0p4p}y)Tm$ru=$o*!8~gNpbQ%J=Y!gW{y53^s z^e^5wY(?+bR(@T`bhd@f0eSW}Y~*Pl#_lm&lwvsmM?7#pY`f~_Ym!L`3IA0=giM(s zEC^JfnVOL6Q~Sdr=n?_8K#VC^vgHS(tJwiSGTv?dQDTi?bZL_&>S`{wXhg>D7`WyZ zd`S?OIky02E=EX}09RbbWVp7kV6zqXvb@bCO#_@5MKXTN$T=%AxwfcdA|=xd;BGfb1KMDfuyR?}EZW4*jSyT1GyA~GMcE5oEl^(G}NTAWsTb208ByJ146|DF5J`ZgBIz^Vm6k!`o9xG@PH&Ls)FHev88?U%_|eJFYas z0^pA!M-=f#jD&b)g_qoq^}4WC$8XNApe03q_nTKB zGXC=4fAgxBM!rgzlAlgo_P%rT8yWV1Q)}eKYfMNtmkbbkKY8FmFjVKS&S6sLA(i}) zN`6QsZ*~HQ3NM$@BP4>h0XB5=KEtc*>g&16Mvar;KX!4fsS$xXzYiDqvDT|Tem7+E z!&I&qtf(4d>HPime|>lQ&Bb@HBQ`V4oHs7^k{>+E*^Z(KF|z#CvGT58so3)hk!K&-Y?ca_@7cvP?_(R{MB{QOE?*R8 zRT)6{nNy9twKVsmxfz*Kj?;dP^gBvC;nYt1u~%TR4fcO5roFXs3;6Hq6$DY@4OeNm zL+61)4uSU)F0g_o!*hdntoqU}?f?MjV_XW*&^|1@zTq}ZCLOZAgeo8TRn=t-;_x!R z=mwByb?5xBC!5H)_K-N{}e&n6gwgTZF{-NajxJOXWqLW$;}Xg!MmSZG1AuA ztfKQ-M&J<#IN4{1n_r{@F}=(SZmloRt;p^Gxs;_<5=pFNlVG1zdPy(OAAR;^sqbYu z6cW4nS}PCDerIo|&J~F|-p~ICU&9d&R-x-CKgW~&eVYWsK5H{Z+{E%za(V|&o_Svzxws-*Zp1o$LYK8 z2u(om!9p3>GwfB^xSEjvef{BmT{EuvEs%*{o(R3cCZ2Fe3#!e~RIhx>1qHuakdkAW zOeErH&X+4#PA-59GoDxg9&}(haC*$H#+(XwZ}s+IraHkeS(~X*m-Q`~GAfa+oQa*) zloJ7&x}Vqzdu~U!q*-=gh02U89hAzm+yo4j2~)^WH92RHWuC0a7}2 zH&S54aIf)|pHCxstY(@tlVTZ?SuL{!i8@hbBQHKv03I;YMpLA+DMKnObkpC|5cfPG z34&DIl$FxoRA%MliUICcbn$GXH}rQ~&k9)Sl7uOwuD@Ycg06r4H(Zg(-sECihb|qE z3&VP8rR5ar`Z^$qo>b8KIn9a{oj|cnFQRVt7tl~xUNgFvju{x9V_m&K9Ca~NL30NA zFF35ou38nB6kiHfq6rUD8c9ayEVCWSGI|Psc3DQOG|WyBx?Do-^*d(sLHR%GI!`M` zt}foWoLEJS=&D&iC>cH;iq4Qztx?PfxlmF=i6Yaw-3sMP!a^Y66p~-#SF6sJz^vI7 zqp3?HG@@ZquyjVqnK_ONo=}Z>ZX{)CWfTy=|BLMIX2pmsQ-(#17Z9`I8x1a1z(V1s zpE=hhE!TKe**%Et%rH7E0u60KzmRxB`mwRxGmH;htS2DTW!4-Dk={dqj+z$a2jYCgLkEqqQpR>Gp@ zxNi`b)D_HeS_mb#%yKZJCxPw9{&?_UN(PytnE1-p&?xpZY$bQ4ReWw)uk~l!)7S|! zdMAOy`rfj>0N(A7PpI+RfW`I|(Q@dbkX>p}hmYIcM=#F~gc|nWH1OWC^^ro_FOb|N5rJ^>Dd>lRP;rI?WSQk(x>K*@O^({stxd#T}*C*K#G;+NVLYjNNZ zp#`DpAe+Ke-0HjheZPZTkxKyL#iiWwYy{W@rdy|l0sMrG%iZkP?wTwcn>O+gZ1-UK zz;6HpsmN%;QU~oq%_>pNmOa6%4?Q_6mtdqxBB07PpPeood>%!vL(&?yJ&#Cr*nDD( z#kZg74lI4@>2TnS145*?y*PZKM_Xc$*}gMZ;6O^-t=wkw*_lco#}i~k`7<)xaDQ-}N$erNT>pqaq~C?N3j$BN(4%mNfpK^USUnX$BG=?S@CNrhsU z&!l*9%hoR@1MCy-3EAz7OYx#M#x|Ix7bYQy@&$Zni@}Z|Q15hVc6w>|yoCCRO~k?H z@hY{Wye;Od`Pv~>S+#ak>mAS`=*NLB4aLDjLqhzn?r@+Tmd8^ptr0fLicLr=#gVoq zrx3rb4Y`wUS^eXPO6>JRO9lu)4vWs(AP~t7Y>WVQ;UfLEbr_rXWIMFkPeKA$0fgVM z(|V(WuA6qt0ljsR4e-`F7YofM|6rvY(Eb*q(okxQ@s`(5OtgRfG)VCyXgGJ)D+i!| zb>hi@IhPW*4k8Yqi8;)Cug9?7$Y*Fa4D9GbZkQfh;LVEFHgdVJ#e~rX@Hi6n2nLlv zPylof4>wuwu$9B)<=&X~_DG^HUW{aTM~fZ2VgB7l&1yQAm3BZ7_c$-OTAb& zBUfhP&TcZ~8gtj71H26@JelDf&1OdyM}D*3Q*=jzjU@~ct{R|0rYd`VfZ|!H8 zVua6Q4{KNhs%&UiR*V4w19ZzGU}QU-QF}e%F{N`Qvr5O7nR=+P@UDYFJM-SK3$ER4 za8rVWf>+lrZSKgpcd^J1wZ;$7OYnJwEpeFjeH6?4P+Q^+ze7VHO~ zCx!fRUG5J2gU^?Q|H0?!&9xr6*_BdvfUm>!!derRB{A11XkyN{YwJ(!(M~v++dS3* z;qpLEuP%46rvz_rx2VM0QG#}!zfI*5%_J@c&2WM9dIFI}vg&7$VE8z#bYqxrjhs^l z)eO4l!wuKcrrXrgla*42>TY#_aC|g5d4#qL{X_u%wV%ei9@Rww&;pG{s$nz0NBWvY zZ_qnRg^*a!MorKU(v#}7?Lm)F0OKH1Qt=eSp}X)Sj|sM;1^>j#7<}(-Ph<6M zJ8tPzvSN8RqR2y!dHt=~wkxY3*a&y9kg>8CU}F0t7n4Tz=Zm}71EOz((LE;iX5psw z=;gj@=t)K#(?dLO^@BbU4>aO!l;#$=!##cCW!gO^$S?0x1^A2;dw|cEu$S#>C!2U_ ztZDZeJb;6p>|iJRN!iH)A;n#9J4ASqmccB9!n9Vi9gd<))}EnN+nLOqy=-vYBxyQ? zj%KqXbP3J8*}T$=>F)+N%S0kmGfpfUS1Ju6d2QFgwb{)gnY+1%PAeGuP|0lPJJ7)j zyd6U<-r-uP#b;ju$S$p~nPlbS8UP{VIk(aHhHo-8P?0guF*Y(wifUpToaHQ+<$B^9 zHD5M*vA;M0F*>Qrk_nxy1A+Pp!$NK9r^zeo0&-wd^pep|f}XjxEO6VTw$zwU znYi1j#XDN6p=W|TR)kLZ9ap<0!BDaOF!(GH#6aHp#%Q*%bIg8&Fs`lWfW8x1>7p8* z?nV41>JGQ7$>k6AZdn-(E$el>QM-N$sx>Rc8}fgT|MM6BoSyvg`1micrvK}&zc~KS z8T{ktCx1Nor~Bgo4O`4>^;xA@mTkN?6h{ux&D|7oX|=>BKJ2?If;&$48R&t#0kjHB^|{~v zh23cb$BH;iJ6O^2#RozX@yl2{`m?v4e--hFwsOlM81V}RBjO$$A`uUfh=ZO(B;p|w z@nB&+SXd8{h^^-kiMV%F=n#n*JcmfcLnPu)I}#BIq!o`iP@l1yXdR6BuN;f0%LkzOJb4AxzSb$rkEUmw= zEK{2Wn2}RbvV<4h0*g@md_adl$`%NIZ3vuY3(vbeKztb(46LZp8qkn4Wrm1?D~PH0 zTrr|nvdU6YvcEeC+0O7L>?`yw4C^$Qid{ik@Yq6Te&oX$8T(xD5-9$fB~qkMVzb-m zw3cqcX^J3N9#i&?W;`{Qp$p!_qc((W@l<-MRP_!LS`?+E$%?69T&R-S${xc}6pE}` za)J6AwC=8csa^c5okV+RH$k`N^#PrtM6Am-l-!bGXE;;72>(z-W~r+U`@2_0)lcx4 z4zj)~z?3!pWkxMAx6s4g{QIxSN>-(sg=G$3!)HTwQ>1l(mnIpbN|V>GNX|v2hhTIB zD&%i|HSm-TQyw3I>@nzMg{mU_+v|(gjEeaepu=qRpF^_#;OlT;|0vQ^tpe6M?lgK=$5ny5HUZW6Bhu4M? z0!o_IGY%rN#v*6U1Qoq*@WG|_p;aR91r{A;WXbQCXw$J(5WbYx?ksJ!uO`--g0S$ev;>NTV=Z8%#xJvM+I%eUQ|LG<`CY=CWNX?TYukVs|`I{OR&7d=SE=q_|5tED#msfC=DCq{@OMwmx60qj$XFT zwTqW{BD2hvVhE#$!pGrg^V*X(xkF<3ff{24Bc9wr9d;Mzz3zO4a!bCzw9g z0HYF`vC9w9Mi~41rlsZDeps+laRq545uRFUve%Vbjlm}xccjen6wy_0N(sTt0s* zuaUPIT@nU=yiB<=|0mbi-+}AD64t=byD~r!_SWV~=<(y^Bb6^r!k}#l=0H#*RqRKE}+;7I<(25~G^j_4GSgA2cnI8LO zgxT1`)8*Y^0ULnP_ib|n><3SmEk9Ove7}vQkv-o~0bl**SHFt5HhBp(J~b2r3F*jos|Vc@pP1XWdCXNdrgxck`^DI$q?ny9vq|s&^?z=1l{u_fIUwE)3bkY zo+p6hc`6{Dy~6KA#(Bg-?RfZ{2z53VU}so5X*8J5m=HPxvw`B_a&}9jdMap~F9wM7 zWnpmk3V!ockT*{NbTb^@W@uhwd`{wS2Wii+H6wu4j0#aR7BI~qb{L-+83#QRnL7Er zG9lyPI0k@l1kz0Ja_n9QY^aNAAJ(6`<*imQ2R!OZ4jrkWMv#_^x`MRPBwWy-uCq1ZSQ<)Z@X$h*gI<} zFgp-?<=ob-!)wAH^3$u!-d>WD0oV)8)NGqP?0j%m%64)v)tco+M8$}Mv;E*~KRDYP z&r`b2R#i^L)bLgC$TwoWTEf7LkjP4)u<&5q2DF7r@En*f#ZaWZYKgQByMyH}aOmSKz@e;Ww%*Le0y51R$$44I z(z-pv>0mS;U#>JOQHRGEtNyco&9tQ=ph7JP6_|}~{_XNZV5;tBnX?tW#ACMzz+fY{l$if~X5$WEa=8Bf-i=evdT;Jfaw zQa+T&d``ZhD#DTWjtaU&``GdI)i>`>3_*b{@A7Walae;2ov#e6)H|_7{Yg;>f z8?kQcsWi(-Vc)IQRGh^&w%2{_YmLyXPT8cED&G#nvgDn-(1{NkMrtWM)21*HauSwN)prS=HfnVJ$Ya@N>g36 z7O+(@-B8Ch{LEUjZ9%^|ju4svF&%%}XzJn3{*EerS`-;)=^4aJ{)wtBysX>*)#d97 zI`}q|bDHg^8w~b5Te0N!Be?uF8}Bk{m5i!wb?qM{-L3n$@gZ(aP} timestamp suffix match > pattern match + framework.GetFrameworkRegistry().Range(func(key, value interface{}) bool { + namespace := key.(string) + fw := value.(*framework.Framework) + + score := 0 + + // Remove timestamp suffix for pattern matching + // e.g., "test-dataflow-1763129228782243000" -> "test-dataflow" + baseNamespace := namespace + if idx := strings.LastIndex(namespace, "-"); idx > 0 { + possibleBase := namespace[:idx] + // Check if suffix is a timestamp (all digits) + suffix := namespace[idx+1:] + isTimestamp := true + for _, c := range suffix { + if c < '0' || c > '9' { + isTimestamp = false + break + } + } + if isTimestamp && len(suffix) > 10 { // timestamps are long + baseNamespace = possibleBase + } + } + + // Extract pattern from namespace: "test-normal-mode" -> "normal mode" (with space) + // This matches Ginkgo's Describe("Normal Mode") to namespace test-normal-mode + namespacePattern := strings.TrimPrefix(baseNamespace, "test-") + namespacePattern = strings.ReplaceAll(namespacePattern, "-", " ") // "normal-mode" -> "normal mode" + + // Scoring: higher score = better match + // Exact namespace match in text - highest priority (1000 points) + if strings.Contains(fullTestPathLower, strings.ToLower(namespace)) { + score += 1000 + } + + // Base namespace match (without timestamp) - very high priority (500 points) + if baseNamespace != namespace && strings.Contains(fullTestPathLower, strings.ToLower(baseNamespace)) { + score += 500 + } + + // Pattern match: "test-normal-mode" matches "Normal Mode" (50 points) + if strings.Contains(fullTestPathLower, strings.ToLower(namespacePattern)) { + score += 50 + } + + // Check if namespace words appear in test path (5 points per word, reduced to avoid false positives) + namespaceWords := strings.Fields(namespacePattern) + for _, word := range namespaceWords { + // Only count meaningful words (skip common words) + if len(word) > 3 && strings.Contains(fullTestPathLower, strings.ToLower(word)) { + score += 5 + } + } + + if score > 0 { + matchedFrameworks = append(matchedFrameworks, fw) + matchScores = append(matchScores, score) + } + + return true // Continue searching all frameworks + }) + + // Select framework with highest match score + if len(matchedFrameworks) > 0 { + bestIdx := 0 + bestScore := matchScores[0] + for i := 1; i < len(matchScores); i++ { + if matchScores[i] > bestScore { + bestScore = matchScores[i] + bestIdx = i + } + } + f = matchedFrameworks[bestIdx] + + if report.Failed() { + fmt.Fprintf(GinkgoWriter, "🔍 Framework matched via registry (fallback) with score %d: namespace=%s\n", + bestScore, f.Namespace()) + } + } + } else { + // Successfully retrieved from report entries (preferred path) + if report.Failed() { + fmt.Fprintf(GinkgoWriter, "✓ Framework retrieved from report entries: namespace=%s\n", + f.Namespace()) + } + } + + if f == nil { + // No framework found - can't collect artifacts + if report.Failed() { + fmt.Fprintf(GinkgoWriter, "⚠️ Cannot collect artifacts: no framework found for test\n") + } + return + } + + // Log artifact collection for failed tests only + if report.Failed() { + fmt.Fprintf(GinkgoWriter, "📦 Collecting artifacts for FAILED test: %s (namespace: %s)\n", + report.LeafNodeText, f.Namespace()) + } + + // Build test info + testInfo := artifacts.TestInfo{ + Name: report.FullText(), + Namespace: f.Namespace(), + Failed: report.Failed(), + FailureMessage: report.FailureMessage(), + Duration: report.RunTime, + StartTime: report.StartTime, + EndTime: report.EndTime, + Labels: report.Labels(), + KubectlClient: f.Kubectl(), + } + + // Collect artifacts with timeout + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := artifactCollector.CollectForTest(ctx, testInfo); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect artifacts: %v\n", err) + } else { + if report.Failed() { + fmt.Fprintf(GinkgoWriter, "✓ Artifacts collected successfully\n") + } + } +}) + +// SynchronizedAfterSuite ensures cleanup runs only once across all parallel processes +var _ = SynchronizedAfterSuite(func() { + // This function runs on ALL processes + // Nothing needed here +}, func() { + // This function runs ONLY on process #1 after all others finish + + // Close artifact collector + if artifactCollector != nil { + if err := artifactCollector.Close(); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to close artifact collector: %v\n", err) + } + } + + // Clean up readiness test namespace if it was created + // We defer this cleanup until after all tests to avoid controller overload + // during test execution (namespace deletion triggers cascading reconciliations) + if readinessTestNamespace != "" { + By("cleaning up controller readiness test namespace") + cmd := exec.Command("kubectl", "delete", "namespace", readinessTestNamespace, "--timeout=30s") + if _, err := utils.Run(cmd); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to delete readiness test namespace %s: %v\n", readinessTestNamespace, err) + } + } + + // Uninstall shared dependencies + framework.UninstallSharedDependencies() + + By("undeploying operator via Helm") + cmd := exec.Command("make", "undeploy-helm-e2e", + fmt.Sprintf("NAMESPACE=%s", operatorNamespace), + ) + _, _ = utils.Run(cmd) +}) + +// verifyControllersReady ensures controllers are fully initialized by creating a test resource +// and verifying the controller processes it. This prevents flaky tests caused by controllers +// still initializing (leader election, cache sync, informers) when pod becomes Ready. +func verifyControllersReady() { + const ( + testNamespace = "controller-readiness-test" + testAggregator = "readiness-test-aggregator" + readinessTimeout = 60 * time.Second + pollInterval = 2 * time.Second + ) + + // Store namespace name for cleanup in AfterSuite + // We don't clean up immediately to avoid controller overload during test execution + // (namespace deletion triggers cascading reconciliations that can interfere with tests) + readinessTestNamespace = testNamespace + + // Create temporary namespace for readiness test (idempotent) + // First delete if exists and wait for full deletion + deleteCmd := exec.Command("kubectl", "delete", "namespace", testNamespace, "--ignore-not-found=true", "--wait=true", "--timeout=30s") + _ = deleteCmd.Run() // Ignore errors, namespace might not exist + + // Now create the namespace + cmd := exec.Command("kubectl", "create", "namespace", testNamespace) + if _, err := utils.Run(cmd); err != nil { + Fail(fmt.Sprintf("Failed to create readiness test namespace: %v", err)) + } + + // Create a minimal VectorAggregator CR + aggregatorYAML := fmt.Sprintf(`apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorAggregator +metadata: + name: %s + namespace: %s +spec: + selector: {} + replicas: 1 + image: timberio/vector:0.40.0-alpine +`, testAggregator, testNamespace) + + cmd = exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(aggregatorYAML) + if _, err := utils.Run(cmd); err != nil { + Fail(fmt.Sprintf("Failed to create test VectorAggregator: %v", err)) + } + + // Wait for controller to create the deployment + deploymentName := testAggregator + "-aggregator" + startTime := time.Now() + + Eventually(func() error { + cmd := exec.Command("kubectl", "get", "deployment", deploymentName, + "-n", testNamespace, "-o", "name") + output, err := utils.Run(cmd) + if err != nil { + return fmt.Errorf("deployment not found: %w", err) + } + if !strings.Contains(string(output), "deployment") { + return fmt.Errorf("deployment not found") + } + return nil + }, readinessTimeout, pollInterval).Should(Succeed(), + "VectorAggregator controller should create deployment %s in namespace %s within %v. "+ + "This indicates controller is not ready. Pod may be Ready but controllers are still initializing "+ + "(leader election, cache sync, webhook registration, informers startup).", + deploymentName, testNamespace, readinessTimeout) + + elapsed := time.Since(startTime) + fmt.Fprintf(GinkgoWriter, "✓ Controllers ready in %.2fs (deployment %s created)\n", + elapsed.Seconds(), deploymentName) +} + // Run e2e tests using the Ginkgo runner. func TestE2E(t *testing.T) { RegisterFailHandler(Fail) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 901f0ade..9dc7e4ea 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -30,63 +30,29 @@ import ( const namespace = "vector-operator-system" var _ = Describe("controller", Ordered, func() { - BeforeAll(func() { - By("installing prometheus operator") - Expect(utils.InstallPrometheusOperator()).To(Succeed()) - - By("installing the cert-manager") - Expect(utils.InstallCertManager()).To(Succeed()) + // NOTE: Dependencies (Prometheus Operator, cert-manager) are installed once + // in BeforeSuite via framework.InstallSharedDependencies() and shared across all tests. + // No need for BeforeAll/AfterAll here. - By("creating manager namespace") + BeforeAll(func() { + By("creating manager namespace (if not exists)") cmd := exec.Command("kubectl", "create", "ns", namespace) - _, _ = utils.Run(cmd) - }) - - AfterAll(func() { - By("uninstalling the Prometheus manager bundle") - utils.UninstallPrometheusOperator() - - By("uninstalling the cert-manager bundle") - utils.UninstallCertManager() - - By("removing manager namespace") - cmd := exec.Command("kubectl", "delete", "ns", namespace) - _, _ = utils.Run(cmd) + _, _ = utils.Run(cmd) // Ignore error if already exists }) Context("Operator", func() { It("should run successfully", func() { var controllerPodName string - var err error - - // projectimage stores the name of the image used in the example - var projectimage = "example.com/vector-operator:v0.0.1" - - By("building the manager(Operator) image") - cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - By("loading the the manager(Operator) image on Kind") - err = utils.LoadImageToKindClusterWithName(projectimage) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("installing CRDs") - cmd = exec.Command("make", "install") - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - - By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage)) - _, err = utils.Run(cmd) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) + // NOTE: Operator is deployed once in BeforeSuite (e2e_suite_test.go) via Helm + // This test verifies that the already-deployed operator is running correctly By("validating that the controller-manager pod is running as expected") verifyControllerUp := func() error { - // Get pod name + // Get pod name (Helm deployment uses different labels) - cmd = exec.Command("kubectl", "get", - "pods", "-l", "control-plane=controller-manager", + cmd := exec.Command("kubectl", "get", + "pods", "-l", "app.kubernetes.io/name=vector-operator", "-o", "go-template={{ range .items }}"+ "{{ if not .metadata.deletionTimestamp }}"+ "{{ .metadata.name }}"+ @@ -101,7 +67,7 @@ var _ = Describe("controller", Ordered, func() { return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames)) } controllerPodName = podNames[0] - ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager")) + ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("vector-operator")) // Validate pod status cmd = exec.Command("kubectl", "get", diff --git a/test/e2e/framework/README.md b/test/e2e/framework/README.md new file mode 100644 index 00000000..b40a9aaf --- /dev/null +++ b/test/e2e/framework/README.md @@ -0,0 +1,612 @@ +# E2E Test Framework + +A comprehensive testing framework for Vector Operator e2e tests, built on top of Ginkgo/Gomega. + +## Overview + +This framework provides a high-level API for writing maintainable and readable e2e tests. It handles common operations like namespace management, resource deployment, status checking, and cleanup, while providing custom matchers for intuitive assertions. + +## Key Features + +- **High-level API** - Simple methods for common operations +- **Automatic namespace management** - Creates and cleans up test namespaces +- **Shared dependencies** - Install Prometheus Operator and cert-manager once for all tests +- **Custom Gomega matchers** - Readable DSL-style assertions +- **Test metrics tracking** - Automatic timing measurements +- **YAML templating** - Dynamic test data generation +- **Centralized timeouts** - Consistent timeout configuration + +## Quick Start + +### Basic Test Structure + +```go +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kaasops/vector-operator/test/e2e/framework" + "github.com/kaasops/vector-operator/test/e2e/framework/assertions" + "github.com/kaasops/vector-operator/test/e2e/framework/config" +) + +var _ = Describe("My Feature", Label(config.LabelSmoke, config.LabelFast), Ordered, func() { + f := framework.NewFramework("my-feature-test") + + BeforeAll(f.Setup) + AfterAll(f.Teardown) + + Context("Basic Functionality", func() { + It("should work correctly", func() { + // Deploy resources + f.ApplyTestData("normal-mode/agent.yaml") + f.ApplyTestData("normal-mode/pipeline-basic.yaml") + + // Wait for readiness + f.WaitForPipelineValid("basic-pipeline") + + // Assert using custom matchers + Eventually(f.Pipeline("basic-pipeline")).Should(assertions.BeValid()) + }) + }) +}) +``` + +## Core Components + +### 1. Framework Object + +The main entry point for all test operations. + +```go +// Create a new framework instance +f := framework.NewFramework("test-namespace-prefix") + +// Setup creates namespace, initializes metrics, and registers framework +// for artifact collection via Ginkgo report entries +f.Setup() + +// Teardown cleans up namespace and resources +f.Teardown() +``` + +**Framework Registration** + +The framework uses Ginkgo's report entry system for context propagation instead of global state: + +```go +// In your test: +f := framework.NewFramework("test-ns") +f.Setup() // Automatically stores framework in Ginkgo report entries + +// In ReportAfterEach (for artifact collection): +// Framework is automatically retrieved from report entries +f := framework.FromReportEntries(report.ReportEntries) +if f != nil { + // Collect artifacts using framework's kubectl client and namespace +} +``` + +**Benefits:** +- ✅ No global state - eliminates race conditions in parallel tests +- ✅ Direct association between test and framework +- ✅ Works correctly with Ginkgo's parallel execution +- ✅ Backward compatible - still supports legacy registry-based matching as fallback + +**Context Support (Advanced)** + +For advanced use cases, the framework can be stored in Go contexts: + +```go +// Store in context +ctx := f.ToContext(context.Background()) + +// Retrieve from context +f := framework.FromContext(ctx) +if f != nil { + // Use framework +} +``` + +### 2. Resource Management + +#### Apply Test Data + +```go +// Load and apply YAML from test/e2e/testdata/ +f.ApplyTestData("normal-mode/agent.yaml") +f.ApplyTestData("normal-mode/pipeline-basic.yaml") +``` + +#### Create Multiple Resources + +```go +// Create 100 pipelines from template +creationTime := f.CreateMultiplePipelinesFromTemplate( + "scalability/pipeline-template.yaml", + "pipeline-NNNN", // Placeholder to replace + 100, // Count +) +fmt.Printf("Created 100 pipelines in %v\n", creationTime) +``` + +### 3. Wait Operations + +```go +// Wait for deployment to be ready (uses config.DeploymentReadyTimeout) +f.WaitForDeploymentReady("aggregator-name") + +// Wait for pipeline to become valid (uses config.PipelineValidTimeout) +f.WaitForPipelineValid("pipeline-name") +``` + +### 3.1. Log Polling Methods + +Standardized methods for waiting on log content, eliminating boilerplate Eventually blocks: + +```go +// Wait for substring to appear in pod logs +err := f.WaitForLogsContaining("pod-name", "expected text", 2*time.Minute) +Expect(err).NotTo(HaveOccurred()) + +// Wait for regex pattern to match in pod logs +err := f.WaitForLogsMatching("pod-name", `\d+ requests processed`, 1*time.Minute) +Expect(err).NotTo(HaveOccurred()) + +// Verify substring does NOT appear in logs (negative assertion) +err := f.AssertNoLogsContaining("pod-name", "ERROR", 30*time.Second) +Expect(err).NotTo(HaveOccurred()) + +// Get logs with options +logs, err := f.GetPodLogsWithOptions("pod-name", framework.LogOptions{ + TailLines: 100, +}) +Expect(err).NotTo(HaveOccurred()) +``` + +**Before (verbose):** +```go +var logs string +Eventually(func() bool { + l, err := f.GetPodLogs("pod-name") + if err != nil { + return false + } + logs = l + return strings.Contains(logs, "expected text") +}, 2*time.Minute, 1*time.Second).Should(BeTrue()) +``` + +**After (concise):** +```go +err := f.WaitForLogsContaining("pod-name", "expected text", 2*time.Minute) +Expect(err).NotTo(HaveOccurred()) +``` + +### 4. Status Queries + +```go +// Get pipeline status field +role := f.GetPipelineStatus("my-pipeline", "role") + +// Count valid pipelines +validCount, err := f.CountValidPipelines() + +// Count services with label +serviceCount := f.CountServicesWithLabel("app.kubernetes.io/component=Aggregator") +``` + +### 5. Custom Matchers + +The framework provides custom Gomega matchers for readable assertions: + +#### Pipeline Matchers + +```go +// Check if pipeline is valid +Eventually(f.Pipeline("test-pipeline")).Should(assertions.BeValid()) +Eventually(f.Pipeline("test-pipeline")).Should(assertions.BeInvalid()) + +// Check role +Expect(f.Pipeline("test-pipeline")).To(assertions.HaveRole("agent")) +Expect(f.Pipeline("test-pipeline")).To(assertions.HaveRole("aggregator")) + +// Check error message contains substring +Expect(f.Pipeline("invalid-pipeline")).To(assertions.HaveErrorContaining("validation")) +``` + +#### Service Matchers + +```go +// Check if service exists +Eventually(f.Service("my-service")).Should(assertions.Exist()) + +// Check service port +Expect(f.Service("my-service")).To(assertions.HavePort("9090")) +``` + +## Shared Dependencies + +Shared dependencies (Prometheus Operator, cert-manager) are installed once in `BeforeSuite` and shared across all tests. + +### Installation + +Handled automatically in `test/e2e/e2e_suite_test.go`: + +```go +var _ = BeforeSuite(func() { + // ... operator deployment + + // Install shared dependencies once + framework.InstallSharedDependencies() +}) +``` + +### Benefits + +- **Faster test execution** - ~3 minutes saved per test run +- **More stable** - Avoid repeated install/uninstall cycles +- **Cleaner logs** - No AlreadyExists errors + +### Usage in Tests + +Tests automatically use shared dependencies: + +```go +// No need to install/uninstall in individual tests +var _ = Describe("My Test", func() { + f := framework.NewFramework("test-ns") + + BeforeAll(f.Setup) // Just creates namespace + AfterAll(f.Teardown) // Just cleans up namespace + + // Dependencies are already available +}) +``` + +## Test Labels + +Ginkgo v2 provides a powerful label system for categorizing and filtering tests. Labels are simply strings that can be attached to test specs. + +### Standard Labels (defined in `config/constants.go`) + +```go +Label(config.LabelSmoke) // Quick smoke tests +Label(config.LabelFast) // Fast tests (<2 min) +Label(config.LabelSlow) // Slow tests (>5 min) +Label(config.LabelStress) // Stress/load tests +Label(config.LabelRegression) // Regression tests +``` + +### Priority Labels + +```go +Label(config.LabelP0) // P0: Critical, must always pass +Label(config.LabelP1) // P1: High priority +Label(config.LabelP2) // P2: Medium priority + +// Example usage: +var _ = Describe("Source Type Constraints [P0-Security]", + Label(config.LabelConstraint, config.LabelP0, config.LabelSecurity, config.LabelFast), func() { + // ... +}) +``` + +### Category Labels + +```go +Label(config.LabelSecurity) // Security-related tests +Label(config.LabelConstraint) // Constraint validation tests +``` + +### Combined Labels + +```go +// Multiple labels for fine-grained filtering +Label(config.LabelSmoke, config.LabelFast) // Quick smoke test +Label(config.LabelP0, config.LabelSecurity, config.LabelFast) // Critical security test +Label(config.LabelStress, config.LabelSlow) // Long-running load test +``` + +### Filtering Tests + +Run specific test categories: + +```bash +# Run only smoke tests +ginkgo --label-filter=smoke ./test/e2e/ + +# Run fast tests +ginkgo --label-filter=fast ./test/e2e/ + +# Exclude slow tests +ginkgo --label-filter="!slow" ./test/e2e/ + +# Run critical security tests +ginkgo --label-filter="p0 && security" ./test/e2e/ + +# Run smoke tests but exclude slow ones +ginkgo --label-filter="smoke && !slow" ./test/e2e/ + +# Run either constraint or security tests +ginkgo --label-filter="constraint || security" ./test/e2e/ +``` + +### Best Practices + +1. **Use descriptive labels**: Labels should clearly indicate what they categorize +2. **Combine standard + custom labels**: Mix project-standard labels with feature-specific ones +3. **Document critical labels**: If using priority labels (P0, P1), document their meaning +4. **Keep labels in test names**: Add labels to Describe text for better readability (e.g., `[P0-Security]`) + +### Available Labels + +List all labels in the test suite: +```bash +ginkgo labels ./test/e2e/ +``` + +## Test Metrics + +The framework automatically tracks test operation timing: + +```go +// Metrics are collected automatically +f.Setup() // Tracks setup time +f.WaitForDeploymentReady(...) // Tracks deployment wait time +f.WaitForPipelineValid(...) // Tracks pipeline validation time +f.Teardown() // Tracks cleanup time + +// Metrics are printed after each test +// Example output: +// 📊 Test Metrics: +// Setup: 60.777ms +// Deployment Wait: 4.299s +// Pipeline Validation: 5.098s +// Cleanup: 11.034s +// Total: 20.472s +``` + +## Environment Variables + +The framework supports several environment variables for customization: + +### E2E_TESTDATA_PATH + +Customize the location of test data files. Defaults to `test/e2e/testdata`. + +```bash +# Use custom test data directory +E2E_TESTDATA_PATH=/path/to/testdata make test-e2e + +# Run tests with test data in a different location +E2E_TESTDATA_PATH=/tmp/my-testdata ginkgo test/e2e/ +``` + +**Use cases:** +- Testing with different data sets +- CI/CD pipelines with mounted test data +- Temporary test data generation +- Isolated test environments + +### E2E_DRY_RUN + +Run tests in dry-run mode to generate test plans without executing them. + +```bash +E2E_DRY_RUN=true make test-e2e +``` + +### E2E_RECORD_STEPS + +Record test steps for debugging and reproducibility. + +```bash +E2E_RECORD_STEPS=true make test-e2e +``` + +## Timeouts Configuration + +Centralized timeout configuration in `config/timeouts.go`: + +```go +const ( + DeploymentCreateTimeout = 90 * time.Second // Wait for deployment to be created + DeploymentReadyTimeout = 120 * time.Second // Wait for deployment to be ready + PipelineValidTimeout = 2 * time.Minute // Wait for pipeline validation + ServiceCreateTimeout = 2 * time.Minute // Wait for service creation + DefaultPollInterval = 2 * time.Second // Default polling interval + SlowPollInterval = 5 * time.Second // Slower polling for heavy ops +) +``` + +## Advanced Examples + +### Example 1: Basic Pipeline Test + +```go +It("should create and validate a basic pipeline with agent", func() { + // Deploy resources + f.ApplyTestData("normal-mode/agent.yaml") + f.ApplyTestData("normal-mode/pipeline-basic.yaml") + + // Wait for readiness + f.WaitForPipelineValid("basic-pipeline") + + // Verify pipeline configuration + Eventually(f.Pipeline("basic-pipeline")).Should(assertions.BeValid()) + Expect(f.Pipeline("basic-pipeline")).To(assertions.HaveRole("agent")) + + // Verify agent processes the pipeline + Eventually(func() error { + return f.VerifyAgentHasPipeline("normal-agent", "basic-pipeline") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) +}) +``` + +### Example 2: Aggregator Test + +```go +It("should deploy aggregator and process pipelines", func() { + // Deploy aggregator + f.ApplyTestData("normal-mode/aggregator.yaml") + f.WaitForDeploymentReady("my-aggregator-aggregator") + + // Create pipeline with aggregator role + f.ApplyTestData("normal-mode/pipeline-aggregator-role.yaml") + f.WaitForPipelineValid("aggregator-pipeline") + + // Verify role + Expect(f.Pipeline("aggregator-pipeline")).To(assertions.HaveRole("aggregator")) +}) +``` + +### Example 3: Scalability Test + +```go +It("should handle 100 pipelines successfully", func() { + const pipelineCount = 100 + + // Deploy aggregator + f.ApplyTestData("scalability/aggregator.yaml") + f.WaitForDeploymentReady("scale-aggregator-aggregator") + + // Create 100 pipelines from template + creationTime := f.CreateMultiplePipelinesFromTemplate( + "scalability/pipeline-template.yaml", + "pipeline-NNNN", + pipelineCount, + ) + GinkgoWriter.Printf("✨ Created %d pipelines in %v\n", pipelineCount, creationTime) + + // Wait for all to become valid (with progress logging) + Eventually(func() (int, error) { + validCount, err := f.CountValidPipelines() + if validCount > 0 { + GinkgoWriter.Printf("📊 Validation progress: %d/%d pipelines valid (%.0f%%)\n", + validCount, pipelineCount, float64(validCount)/float64(pipelineCount)*100) + } + return validCount, nil + }, 7*time.Minute, 10*time.Second).Should(Equal(pipelineCount)) +}) +``` + +## Best Practices + +### 1. Use Descriptive Test Names + +```go +// Good +It("should create and validate a basic pipeline with agent", func() { ... }) + +// Bad +It("test1", func() { ... }) +``` + +### 2. Use Eventually for Async Operations + +```go +// Good - waits for condition to be met +Eventually(f.Pipeline("test-pipeline")).Should(assertions.BeValid()) + +// Bad - may fail if not ready immediately +Expect(f.Pipeline("test-pipeline")).To(assertions.BeValid()) +``` + +### 3. Use Appropriate Labels + +```go +// Mark fast smoke tests +var _ = Describe("Quick Validation", Label(config.LabelSmoke, config.LabelFast), ...) + +// Mark slow stress tests +var _ = Describe("Load Test", Label(config.LabelStress, config.LabelSlow), ...) +``` + +### 4. Leverage Test Metrics + +```go +// Metrics are automatically tracked and displayed +BeforeAll(f.Setup) // Tracks setup time +AfterAll(f.Teardown) // Tracks cleanup time + displays all metrics +``` + +### 5. Use Custom Matchers + +```go +// Good - readable and clear intent +Expect(f.Pipeline("test")).To(assertions.BeValid()) +Expect(f.Pipeline("test")).To(assertions.HaveRole("agent")) + +// Bad - verbose and less clear +status := f.GetPipelineStatus("test", "configCheckResult") +Expect(status).To(Equal("true")) +role := f.GetPipelineStatus("test", "role") +Expect(role).To(Equal("agent")) +``` + +## Directory Structure + +``` +test/e2e/framework/ +├── README.md # This file +├── framework.go # Main framework implementation +├── lifecycle.go # Shared dependencies management +├── resources.go # Resource utilities +├── config/ +│ ├── constants.go # Test labels and constants +│ └── timeouts.go # Timeout configuration +├── kubectl/ +│ ├── client.go # Kubectl wrapper +│ ├── wait.go # Wait utilities +│ └── validation.go # Validation helpers +├── assertions/ +│ └── matchers.go # Custom Gomega matchers +├── artifacts/ +│ ├── collector.go # Artifact collection +│ ├── storage.go # Artifact storage +│ └── config.go # Artifact configuration +├── errors/ +│ └── errors.go # Custom error types +└── recorder/ + └── recorder.go # Step recorder +``` + +## Contributing + +When adding new features to the framework: + +1. Keep the API simple and intuitive +2. Add appropriate error handling +3. Track timing metrics for new operations +4. Add custom matchers for common assertions +5. Update this README with examples + +## Troubleshooting + +### AlreadyExists Errors + +If you see `AlreadyExists` errors for Prometheus Operator or cert-manager: +- Ensure you're not installing dependencies in `BeforeAll` +- Dependencies are automatically installed in `BeforeSuite` via `framework.InstallSharedDependencies()` + +### Timeout Errors + +If tests timeout: +- Check `config/timeouts.go` and adjust as needed +- Use `SlowPollInterval` for expensive operations +- Consider increasing go test timeout: `-timeout=15m` + +### Namespace Not Found + +If you see namespace errors: +- Ensure `BeforeAll(f.Setup)` is called +- Verify namespace name matches test data YAML files + +## References + +- [Ginkgo Documentation](https://onsi.github.io/ginkgo/) +- [Gomega Matchers](https://onsi.github.io/gomega/) +- [Vector Operator E2E Tests](../README.md) diff --git a/test/e2e/framework/artifacts/README.md b/test/e2e/framework/artifacts/README.md new file mode 100644 index 00000000..97b774d5 --- /dev/null +++ b/test/e2e/framework/artifacts/README.md @@ -0,0 +1,178 @@ +# E2E Test Artifact Collection + +Automatic collection of debugging artifacts when e2e tests fail. + +## Features + +- **Automatic**: Collects artifacts only on test failures (configurable) +- **Safe**: Never fails tests due to collection errors +- **Fast**: < 1s overhead for passing tests, < 30s for failing tests +- **Comprehensive**: Logs, pod status, events, resources + +## Configuration (ENV Variables) + +All configuration is ENV-based with sensible defaults: + +### Collection Control +- `E2E_ARTIFACTS_ENABLED` (default: `true`) - Master switch +- `E2E_ARTIFACTS_ON_FAILURE_ONLY` (default: `true`) - Only collect on failures +- `E2E_ARTIFACTS_MINIMAL_ONLY` (default: `false`) - P0 artifacts only + +### Storage +- `E2E_ARTIFACTS_DIR` (default: `test/e2e/artifacts`) - Base directory + +### Size Limits +- `E2E_ARTIFACTS_MAX_LOG_LINES` (default: `500`) - Max log lines per pod +- `E2E_ARTIFACTS_MAX_RESOURCE_SIZE` (default: `10485760`) - Max 10MB per file +- `E2E_ARTIFACTS_MAX_TOTAL_SIZE` (default: `104857600`) - Max 100MB per test + +### Timeouts +- `E2E_ARTIFACTS_TIMEOUT` (default: `30s`) - Max collection time + +## Usage + +### Running Tests with Artifacts + +```bash +# Default behavior (enabled, on-failure-only) +make test-e2e + +# Disable artifacts +E2E_ARTIFACTS_ENABLED=false make test-e2e + +# Collect for all tests (even passing) +E2E_ARTIFACTS_ON_FAILURE_ONLY=false make test-e2e + +# Increase log lines +E2E_ARTIFACTS_MAX_LOG_LINES=1000 make test-e2e +``` + +### Artifact Location + +Artifacts are stored in: +``` +test/e2e/artifacts/ +└── run-{timestamp}/ + ├── metadata.json + └── {test-name}/ + ├── metadata.json + ├── logs/ + │ ├── operator-controller.log + │ └── pod-{name}.log + ├── pods/ + │ └── {pod-name}-status.json + ├── resources/ + │ ├── vectorpipeline-{name}-status.json + │ └── deployment-{name}.yaml + └── events/ + └── namespace-events.txt +``` + +### Unified Test Results + +When you run `make test-e2e`, all results are automatically saved in a unified structure with reports and artifacts correlated by timestamp: + +```bash +# Run tests - results automatically saved with timestamp +make test-e2e + +# Results structure: +test/e2e/results/run-{timestamp}/ +├── reports/ +│ ├── junit-report.xml # JUnit XML for CI integration +│ ├── report.json # Ginkgo JSON report +│ └── test-output.log # Full test output logs +└── artifacts/ # Debug artifacts (only for failed tests) + ├── metadata.json # Run-level metadata + └── {test-name}/ # Per-test artifacts + ├── metadata.json + ├── logs/ + ├── pods/ + ├── resources/ + └── events/ +``` + +**Benefits**: +- Single runID correlates all reports and artifacts +- Easy to navigate - everything in one directory +- CI/CD friendly - upload one directory +- Helpful output with quick analysis commands + +### CI Integration (GitHub Actions) + +```yaml +- name: Run E2E Tests + run: make test-e2e + +- name: Upload Test Results + if: always() # Upload even if tests fail + uses: actions/upload-artifact@v4 + with: + name: e2e-results-${{ github.run_number }} + path: test/e2e/results/ + retention-days: 30 +``` + +## Collected Artifacts (P0 - MVP) + +### Critical for Debugging +1. **Pod Status JSON** - Conditions, restarts, phase +2. **Operator Controller Logs** - Time-filtered logs (test duration + 1min buffer) +3. **VectorPipeline CR Status** - Validation results +4. **Namespace Events** - What happened in test namespace +5. **Resource Metadata** - Deployments, DaemonSets, Services + +### Future (Phase 2) +- Full pod logs (all containers) +- Full pod descriptions +- Vector agent/aggregator logs +- ConfigCheck pod logs +- Timeline reconstruction + +## Architecture + +- **Thread-safe**: Uses `sync.Map` for parallel test support +- **Graceful degradation**: Collection errors don't fail tests +- **Size limits**: Prevents CI artifact bloat +- **Atomic writes**: Temp file + rename for reliability + +## Performance + +- **Passing tests**: < 1s overhead (if `ON_FAILURE_ONLY=true`) +- **Failing tests**: < 30s collection time +- **Storage**: < 100MB per test, < 500MB per run + +## Troubleshooting + +### No artifacts collected +1. Check `E2E_ARTIFACTS_ENABLED=true` +2. Verify test is using `framework.NewFramework()` or `framework.Shared()` +3. Check GinkgoWriter output for warning messages + +### Artifacts too large +1. Reduce `E2E_ARTIFACTS_MAX_LOG_LINES` (default: 500) +2. Enable `E2E_ARTIFACTS_MINIMAL_ONLY=true` +3. Check individual file sizes with `E2E_ARTIFACTS_MAX_RESOURCE_SIZE` + +### Collection timeout +1. Increase `E2E_ARTIFACTS_TIMEOUT` (default: 30s) +2. Check kubectl connectivity +3. Review namespace resource count + +## Important Bug Fixes + +### Time-based Log Collection (Fixed) +**Problem**: Previously, operator logs were collected using `kubectl logs --tail 500`, which retrieved the last 500 lines from the entire pod lifetime. In long-running test suites (e.g., full e2e runs lasting 15+ minutes), the operator pod could generate thousands of log lines, causing the last 500 lines to exclude logs from earlier failing tests. + +**Example**: A test failing at 18:05-18:07 would collect operator logs from 16:02-16:03 (the pod's startup logs), completely missing the relevant reconciliation attempts. + +**Solution**: Implemented time-based log collection using `kubectl logs --since-time` with the test's start time (+ 1 minute buffer). This ensures operator logs are collected only for the relevant time period, regardless of how long the pod has been running. + +**Impact**: +- Fixes flaky test debugging where operator logs were missing +- Enables reliable root cause analysis for race conditions +- Reduces confusion when logs don't match test timeline + +## Development + +See architect design document for Phase 2+ enhancements. diff --git a/test/e2e/framework/artifacts/collector.go b/test/e2e/framework/artifacts/collector.go new file mode 100644 index 00000000..0be6e612 --- /dev/null +++ b/test/e2e/framework/artifacts/collector.go @@ -0,0 +1,621 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package artifacts + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" //nolint:golint,revive + + "github.com/kaasops/vector-operator/test/e2e/framework/kubectl" +) + +// TestInfo contains information about a test execution +type TestInfo struct { + Name string + Namespace string + Failed bool + FailureMessage string + Duration time.Duration + StartTime time.Time + EndTime time.Time + Labels []string + + // Test sequence tracking (for degradation analysis) + SequenceNumber int // Which test in the run (1, 2, 3...) + OperatorAge time.Duration // How long operator has been running + + // Kubernetes context + KubectlClient *kubectl.Client +} + +// Collector manages artifact collection for e2e tests +type Collector interface { + // Initialize sets up the collector for a test run + Initialize(runID string) error + + // CollectForTest collects artifacts for a test + CollectForTest(ctx context.Context, testInfo TestInfo) error + + // Close finalizes the collector and writes summary + Close() error +} + +// collector implements the Collector interface +type collector struct { + config Config + storage *Storage + metadata *MetadataBuilder + runStart time.Time + + // Operator tracking (for degradation analysis) + operatorStartTime time.Time + + // Statistics + totalTests int + failedTests int + testCounter int // Counter for directory naming +} + +// NewCollector creates a new artifact collector +func NewCollector(config Config) (Collector, error) { + return &collector{ + config: config, + runStart: time.Now(), + }, nil +} + +// Initialize sets up the collector for a test run +func (c *collector) Initialize(runID string) error { + if !c.config.Enabled { + return nil + } + + // Create storage + storage, err := NewStorage(c.config.BaseDir, runID, c.config.MaxResourceSize) + if err != nil { + return fmt.Errorf("failed to create storage: %w", err) + } + c.storage = storage + + // Create metadata builder + c.metadata = NewMetadataBuilder(storage) + + // Get operator start time for degradation tracking + c.operatorStartTime = c.getOperatorStartTime() + + fmt.Fprintf(GinkgoWriter, "📦 Artifact collection initialized: %s\n", storage.GetRunDir()) + return nil +} + +// CollectForTest collects artifacts for a specific test +func (c *collector) CollectForTest(ctx context.Context, testInfo TestInfo) error { + if !c.config.Enabled { + return nil + } + + // Update statistics + c.totalTests++ + if testInfo.Failed { + c.failedTests++ + } + + // Skip passed tests if configured + if !testInfo.Failed && c.config.CollectOnFailureOnly { + return nil + } + + // Increment counter and create short directory name + c.testCounter++ + shortName := createShortTestName(testInfo.Name, c.testCounter) + + // Fill in tracking fields for degradation analysis + testInfo.SequenceNumber = c.totalTests + if !c.operatorStartTime.IsZero() { + testInfo.OperatorAge = time.Since(c.operatorStartTime) + } + + // Create test directory + testDir, err := c.storage.CreateTestDir(shortName) + if err != nil { + return fmt.Errorf("failed to create test directory: %w", err) + } + + fmt.Fprintf(GinkgoWriter, "📦 Collecting artifacts for test: %s\n", testInfo.Name) + collectionStart := time.Now() + + // Collect with timeout + ctx, cancel := context.WithTimeout(ctx, c.config.CollectionTimeout) + defer cancel() + + // Track collected artifacts + inventory := ArtifactInventory{ + LogFiles: []string{}, + ResourceFiles: []string{}, + EventFiles: []string{}, + } + + // P0 artifacts - critical for debugging + if err := c.collectP0Artifacts(ctx, testInfo, testDir, &inventory); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Warning: P0 artifact collection had errors: %v\n", err) + } + + // Write test metadata + collectionDuration := time.Since(collectionStart) + inventory.CollectionTime = collectionDuration.String() + + meta := BuildTestMetadata(testInfo, inventory) + if err := c.metadata.WriteTestMetadata(meta, testDir); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Warning: Failed to write test metadata: %v\n", err) + } + + fmt.Fprintf(GinkgoWriter, "✅ Artifacts collected in %v (%d files)\n", + collectionDuration, len(inventory.LogFiles)+len(inventory.ResourceFiles)+len(inventory.EventFiles)) + + return nil +} + +// collectP0Artifacts collects P0 (critical) artifacts +func (c *collector) collectP0Artifacts(ctx context.Context, testInfo TestInfo, testDir string, inventory *ArtifactInventory) error { + kubectl := testInfo.KubectlClient + namespace := testInfo.Namespace + + if kubectl == nil || namespace == "" { + return fmt.Errorf("missing kubectl client or namespace") + } + + // 1. Pod status (JSON) - fast, critical + if err := c.collectPodStatus(ctx, kubectl, namespace, testDir, inventory); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect pod status: %v\n", err) + } + + // 2. Operator controller logs - critical for debugging + if err := c.collectOperatorLogs(ctx, testDir, inventory, testInfo.StartTime); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect operator logs: %v\n", err) + } + + // 2a. Operator health (pod describe, events) - critical for degradation diagnosis + if err := c.collectOperatorHealth(ctx, testDir, inventory, testInfo.StartTime); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect operator health: %v\n", err) + } + + // 3. Pipeline status - fast, shows validation state + if err := c.collectPipelineStatus(ctx, kubectl, namespace, testDir, inventory); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect pipeline status: %v\n", err) + } + + // 4. Namespace events - fast, shows what happened + if err := c.collectEvents(ctx, kubectl, namespace, testDir, inventory); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect namespace events: %v\n", err) + } + + // 5. Resource metadata (Deployment/DaemonSet/Service basic info) + if err := c.collectResourceMetadata(ctx, kubectl, namespace, testDir, inventory); err != nil { + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect resource metadata: %v\n", err) + } + + return nil +} + +// collectPodStatus collects status of all pods in the namespace +func (c *collector) collectPodStatus(ctx context.Context, kubectl *kubectl.Client, namespace, testDir string, inventory *ArtifactInventory) error { + // Get all pods + pods, err := kubectl.GetPodsByLabel("") + if err != nil { + return fmt.Errorf("failed to get pods: %w", err) + } + + inventory.PodCount = len(pods) + + for _, podName := range pods { + // Get pod status as JSON + output, err := kubectl.GetWithJsonPath("pod", podName, ".status") + if err != nil { + fmt.Fprintf(GinkgoWriter, " ⚠️ Failed to get status for pod %s: %v\n", podName, err) + continue + } + + filename := fmt.Sprintf("%s-status.json", podName) + if err := c.storage.WriteFile(testDir, "pods", filename, []byte(output)); err != nil { + fmt.Fprintf(GinkgoWriter, " ⚠️ Failed to write pod status for %s: %v\n", podName, err) + continue + } + + inventory.ResourceFiles = append(inventory.ResourceFiles, "pods/"+filename) + + // Also get pod logs (last N lines) + logs, err := kubectl.GetPodLogsTail(podName, c.config.MaxLogLines) + if err != nil { + // Pod might not have logs yet, that's okay + continue + } + + // Truncate logs if needed + truncatedLogs := TruncateLogLines([]byte(logs), c.config.MaxLogLines) + + logFilename := fmt.Sprintf("%s.log", podName) + if err := c.storage.WriteFile(testDir, "logs", logFilename, truncatedLogs); err != nil { + fmt.Fprintf(GinkgoWriter, " ⚠️ Failed to write logs for %s: %v\n", podName, err) + continue + } + + inventory.LogFiles = append(inventory.LogFiles, "logs/"+logFilename) + } + + return nil +} + +// collectOperatorLogs collects operator controller logs +func (c *collector) collectOperatorLogs(ctx context.Context, testDir string, inventory *ArtifactInventory, testStart time.Time) error { + // Get operator pod name from vector-operator-system namespace + operatorNs := "vector-operator-system" + operatorClient := kubectl.NewClient(operatorNs) + + pods, err := operatorClient.GetPodsByLabel("app.kubernetes.io/name=vector-operator") + if err != nil || len(pods) == 0 { + return fmt.Errorf("failed to find operator controller pod: %w", err) + } + + // Get logs from first controller pod (should only be one) + podName := pods[0] + + // Add 1 minute buffer before test start to capture context + // This helps see what was happening just before the test started + logsSince := testStart.Add(-1 * time.Minute) + + // Use time-based log collection to get logs relevant to this test + // This fixes the issue where long-running operator pods would only return + // the last N lines from the entire pod lifetime, missing test-specific logs + logs, err := operatorClient.GetPodLogsSinceTime(podName, logsSince, c.config.MaxLogLines) + if err != nil { + return fmt.Errorf("failed to get operator logs: %w", err) + } + + // Truncate logs if still needed + truncatedLogs := TruncateLogLines([]byte(logs), c.config.MaxLogLines) + + if err := c.storage.WriteFile(testDir, "logs", "operator-controller.log", truncatedLogs); err != nil { + return fmt.Errorf("failed to write operator logs: %w", err) + } + + inventory.LogFiles = append(inventory.LogFiles, "logs/operator-controller.log") + return nil +} + +// collectOperatorHealth collects operator pod describe and events for degradation diagnosis +func (c *collector) collectOperatorHealth(ctx context.Context, testDir string, inventory *ArtifactInventory, testStart time.Time) error { + const operatorNs = "vector-operator-system" + operatorClient := kubectl.NewClient(operatorNs) + + // Get operator pod + pods, err := operatorClient.GetPodsByLabel("app.kubernetes.io/name=vector-operator") + if err != nil || len(pods) == 0 { + return fmt.Errorf("failed to find operator pod: %w", err) + } + podName := pods[0] + + // 1. Get pod describe (shows conditions, events, restarts, QoS, resource requests/limits) + describeCmd := exec.Command("kubectl", "describe", "pod", podName, "-n", operatorNs) + describeOutput, err := describeCmd.CombinedOutput() + if err == nil { + if err := c.storage.WriteFile(testDir, "operator", "pod-describe.txt", describeOutput); err != nil { + return fmt.Errorf("failed to write operator pod describe: %w", err) + } + inventory.ResourceFiles = append(inventory.ResourceFiles, "operator/pod-describe.txt") + } + + // 2. Get cluster-wide events related to operator (evictions, OOMKills, etc.) + // Use time window from 2 minutes before test start to catch context + sinceTime := testStart.Add(-2 * time.Minute).UTC().Format(time.RFC3339) + + // Get all Warning events in operator namespace + eventsCmd := exec.Command("kubectl", "get", "events", "-n", operatorNs, + "--field-selector", "type=Warning", + "--since-time", sinceTime) + eventsOutput, err := eventsCmd.CombinedOutput() + if err == nil && len(eventsOutput) > 0 { + if err := c.storage.WriteFile(testDir, "operator", "warning-events.txt", eventsOutput); err != nil { + return fmt.Errorf("failed to write operator warning events: %w", err) + } + inventory.EventFiles = append(inventory.EventFiles, "operator/warning-events.txt") + } + + // 3. Get deployment describe (shows replica status, conditions) + deployDescribeCmd := exec.Command("kubectl", "describe", "deployment", "vector-operator-controller-manager", "-n", operatorNs) + deployDescribeOutput, err := deployDescribeCmd.CombinedOutput() + if err == nil { + if err := c.storage.WriteFile(testDir, "operator", "deployment-describe.txt", deployDescribeOutput); err != nil { + return fmt.Errorf("failed to write deployment describe: %w", err) + } + inventory.ResourceFiles = append(inventory.ResourceFiles, "operator/deployment-describe.txt") + } + + // 4. Collect pprof profiles (goroutine, heap) for memory/goroutine leak diagnosis + if err := c.collectPprofProfiles(ctx, testDir, inventory, podName, operatorNs); err != nil { + // Non-fatal: pprof may not be enabled in production + fmt.Fprintf(GinkgoWriter, "⚠️ Failed to collect pprof profiles (may not be enabled): %v\n", err) + } + + return nil +} + +// collectPipelineStatus collects VectorPipeline CR status +func (c *collector) collectPipelineStatus(ctx context.Context, kubectl *kubectl.Client, namespace, testDir string, inventory *ArtifactInventory) error { + // Get all VectorPipeline CRs + pipelinesOutput, err := kubectl.GetAll("vectorpipeline", "") + if err != nil { + return fmt.Errorf("failed to list pipelines: %w", err) + } + + if pipelinesOutput == "" { + // No pipelines, that's okay + return nil + } + + pipelines := strings.Fields(pipelinesOutput) + for _, pipelineName := range pipelines { + // Get pipeline status + status, err := kubectl.GetWithJsonPath("vectorpipeline", pipelineName, ".status") + if err != nil { + fmt.Fprintf(GinkgoWriter, " ⚠️ Failed to get status for pipeline %s: %v\n", pipelineName, err) + continue + } + + filename := fmt.Sprintf("vectorpipeline-%s-status.json", pipelineName) + if err := c.storage.WriteFile(testDir, "resources", filename, []byte(status)); err != nil { + fmt.Fprintf(GinkgoWriter, " ⚠️ Failed to write pipeline status for %s: %v\n", pipelineName, err) + continue + } + + inventory.ResourceFiles = append(inventory.ResourceFiles, "resources/"+filename) + } + + return nil +} + +// collectEvents collects Kubernetes events from the namespace +func (c *collector) collectEvents(ctx context.Context, kubectl *kubectl.Client, namespace, testDir string, inventory *ArtifactInventory) error { + // Get events - use kubectl.Client to run kubectl get events + // Since we don't have a GetEvents method, we'll use a simple approach + eventsOutput, err := kubectl.Get("events", "") + if err != nil { + // Events might not exist, that's okay + return nil + } + + if err := c.storage.WriteFile(testDir, "events", "namespace-events.txt", eventsOutput); err != nil { + return fmt.Errorf("failed to write namespace events: %w", err) + } + + inventory.EventFiles = append(inventory.EventFiles, "events/namespace-events.txt") + return nil +} + +// collectResourceMetadata collects basic metadata about Deployments, DaemonSets, Services +func (c *collector) collectResourceMetadata(ctx context.Context, kubectl *kubectl.Client, namespace, testDir string, inventory *ArtifactInventory) error { + resourceTypes := []string{"deployment", "daemonset", "service"} + + for _, resourceType := range resourceTypes { + resources, err := kubectl.GetAll(resourceType, "") + if err != nil { + continue // Resource type might not exist + } + + if resources == "" { + continue + } + + resourceNames := strings.Fields(resources) + for _, resourceName := range resourceNames { + // Get resource metadata (name, labels, status) + output, err := kubectl.Get(resourceType, resourceName) + if err != nil { + continue + } + + filename := fmt.Sprintf("%s-%s.yaml", resourceType, resourceName) + if err := c.storage.WriteFile(testDir, "resources", filename, output); err != nil { + fmt.Fprintf(GinkgoWriter, " ⚠️ Failed to write %s/%s: %v\n", resourceType, resourceName, err) + continue + } + + inventory.ResourceFiles = append(inventory.ResourceFiles, "resources/"+filename) + } + } + + return nil +} + +// Close finalizes the collector and writes run summary +func (c *collector) Close() error { + if !c.config.Enabled || c.storage == nil { + return nil + } + + runEnd := time.Now() + runMeta := RunMetadata{ + RunID: c.storage.GetRunID(), + StartTime: c.runStart, + EndTime: runEnd, + TotalTests: c.totalTests, + FailedTests: c.failedTests, + PassedTests: c.totalTests - c.failedTests, + ArtifactsDir: c.storage.GetRunDir(), + Environment: map[string]string{ + "E2E_ARTIFACTS_ENABLED": fmt.Sprintf("%t", c.config.Enabled), + "E2E_ARTIFACTS_ON_FAILURE_ONLY": fmt.Sprintf("%t", c.config.CollectOnFailureOnly), + "E2E_ARTIFACTS_MAX_LOG_LINES": fmt.Sprintf("%d", c.config.MaxLogLines), + "E2E_ARTIFACTS_COLLECTION_TIME": runEnd.Sub(c.runStart).String(), + }, + GitCommit: os.Getenv("E2E_GIT_COMMIT"), + GitBranch: os.Getenv("E2E_GIT_BRANCH"), + GitDirty: os.Getenv("E2E_GIT_DIRTY"), + Description: os.Getenv("E2E_RUN_DESCRIPTION"), + } + + if err := c.metadata.WriteRunMetadata(runMeta); err != nil { + return fmt.Errorf("failed to write run metadata: %w", err) + } + + fmt.Fprintf(GinkgoWriter, "\n📦 Artifact Collection Summary:\n") + fmt.Fprintf(GinkgoWriter, " Location: %s\n", c.storage.GetRunDir()) + fmt.Fprintf(GinkgoWriter, " Total tests: %d\n", c.totalTests) + fmt.Fprintf(GinkgoWriter, " Failed tests with artifacts: %d\n", c.failedTests) + fmt.Fprintf(GinkgoWriter, " Duration: %v\n\n", runEnd.Sub(c.runStart)) + + return nil +} + +// createShortTestName creates a short, numbered directory name from full test name +// Input: "Artifact Verification should intentionally fail to test artifact collection" +// Output: "01-artifact-verification" +func createShortTestName(fullName string, counter int) string { + // Split by spaces to get the first part (Describe block name) + parts := strings.Fields(fullName) + if len(parts) == 0 { + return fmt.Sprintf("%02d-unknown", counter) + } + + // Stop at "should" or "[" - these mark the end of test suite name + var suiteParts []string + for _, word := range parts { + lower := strings.ToLower(word) + // Stop at common separators + if lower == "should" || strings.HasPrefix(word, "[") { + break + } + // Clean up and add word + clean := strings.Trim(word, "()[]{}") + if clean != "" { + suiteParts = append(suiteParts, clean) + } + // Limit to first 3-4 words + if len(suiteParts) >= 4 { + break + } + } + + // Fallback if nothing found + if len(suiteParts) == 0 { + suiteParts = parts[:1] + } + + // Join and lowercase + mainPart := strings.ToLower(strings.Join(suiteParts, "-")) + + // Remove any remaining special characters + replacer := strings.NewReplacer( + "(", "", ")", "", + "[", "", "]", "", + "{", "", "}", "", + ) + mainPart = replacer.Replace(mainPart) + + // Limit length to reasonable size + const maxLen = 40 + if len(mainPart) > maxLen { + mainPart = mainPart[:maxLen] + } + + // Add counter prefix for uniqueness and ordering + return fmt.Sprintf("%02d-%s", counter, mainPart) +} + +// getOperatorStartTime retrieves the operator pod's start time for degradation tracking +func (c *collector) getOperatorStartTime() time.Time { + const operatorNs = "vector-operator-system" + operatorClient := kubectl.NewClient(operatorNs) + + // Get operator pods + pods, err := operatorClient.GetPodsByLabel("app.kubernetes.io/name=vector-operator") + if err != nil || len(pods) == 0 { + // If we can't get operator pod, return zero time + return time.Time{} + } + + // Get pod start time + startTimeStr, err := operatorClient.GetWithJsonPath("pod", pods[0], ".status.startTime") + if err != nil { + return time.Time{} + } + + // Parse RFC3339 timestamp + startTime, err := time.Parse(time.RFC3339, strings.TrimSpace(startTimeStr)) + if err != nil { + return time.Time{} + } + + return startTime +} + +// collectPprofProfiles collects pprof profiles from the operator pod for leak diagnosis +// Uses kubectl port-forward since distroless image doesn't have wget/curl +func (c *collector) collectPprofProfiles(ctx context.Context, testDir string, inventory *ArtifactInventory, podName, namespace string) error { + const pprofPort = "6060" + const localPort = "16060" // Use high port to avoid conflicts + + // Start port-forward in background + portForwardCmd := exec.Command("kubectl", "port-forward", + fmt.Sprintf("pod/%s", podName), + "-n", namespace, + fmt.Sprintf("%s:%s", localPort, pprofPort)) + + if err := portForwardCmd.Start(); err != nil { + return fmt.Errorf("failed to start port-forward: %w", err) + } + defer func() { + if portForwardCmd.Process != nil { + _ = portForwardCmd.Process.Kill() + } + }() + + // Wait a bit for port-forward to establish + time.Sleep(2 * time.Second) + + // Collect goroutine profile (text format for readability) + goroutineCmd := exec.Command("curl", "-s", + fmt.Sprintf("http://localhost:%s/debug/pprof/goroutine?debug=1", localPort)) + goroutineOutput, err := goroutineCmd.CombinedOutput() + if err == nil && len(goroutineOutput) > 0 { + if err := c.storage.WriteFile(testDir, "operator", "pprof-goroutine.txt", goroutineOutput); err != nil { + return fmt.Errorf("failed to write goroutine profile: %w", err) + } + inventory.ResourceFiles = append(inventory.ResourceFiles, "operator/pprof-goroutine.txt") + } else { + return fmt.Errorf("failed to collect goroutine profile: %w", err) + } + + // Collect heap profile (text format for readability) + heapCmd := exec.Command("curl", "-s", + fmt.Sprintf("http://localhost:%s/debug/pprof/heap?debug=1", localPort)) + heapOutput, err := heapCmd.CombinedOutput() + if err == nil && len(heapOutput) > 0 { + if err := c.storage.WriteFile(testDir, "operator", "pprof-heap.txt", heapOutput); err != nil { + return fmt.Errorf("failed to write heap profile: %w", err) + } + inventory.ResourceFiles = append(inventory.ResourceFiles, "operator/pprof-heap.txt") + } else { + return fmt.Errorf("failed to collect heap profile: %w", err) + } + + return nil +} diff --git a/test/e2e/framework/artifacts/config.go b/test/e2e/framework/artifacts/config.go new file mode 100644 index 00000000..c6033566 --- /dev/null +++ b/test/e2e/framework/artifacts/config.go @@ -0,0 +1,137 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package artifacts + +import ( + "os" + "strconv" + "time" +) + +// Default configuration values +const ( + defaultBaseDir = "test/e2e/artifacts" + defaultMaxLogLines = 500 + defaultMaxResourceSize = 10 * 1024 * 1024 // 10MB + defaultMaxTotalSize = 100 * 1024 * 1024 // 100MB per test + defaultCollectionTimeout = 30 * time.Second + defaultEnabled = true + defaultOnFailureOnly = true + defaultMinimalOnly = false +) + +// Config defines artifact collection behavior +type Config struct { + // Collection control + Enabled bool // Master switch for artifact collection + CollectOnFailureOnly bool // Collect artifacts only for failed tests + CollectMinimalOnly bool // Collect only P0 artifacts (fast path) + + // Storage paths + BaseDir string // Base directory for artifact storage + + // Size limits (prevent artifact bloat) + MaxLogLines int // Maximum log lines per pod + MaxResourceSize int64 // Maximum size for single resource (bytes) + MaxTotalSize int64 // Maximum total size per test (bytes) + + // Timeouts + CollectionTimeout time.Duration // Maximum time to collect artifacts + + // Filters + NamespacePatterns []string // Namespace patterns to collect from + PodLabelSelectors []string // Pod label selectors for filtering +} + +// LoadConfigFromEnv loads configuration from environment variables +// Following Phase 1 pattern: ENV-based config with sensible defaults +func LoadConfigFromEnv() Config { + return Config{ + Enabled: getEnvBool("E2E_ARTIFACTS_ENABLED", defaultEnabled), + CollectOnFailureOnly: getEnvBool("E2E_ARTIFACTS_ON_FAILURE_ONLY", defaultOnFailureOnly), + CollectMinimalOnly: getEnvBool("E2E_ARTIFACTS_MINIMAL_ONLY", defaultMinimalOnly), + + BaseDir: getEnvString("E2E_ARTIFACTS_DIR", defaultBaseDir), + + MaxLogLines: getEnvInt("E2E_ARTIFACTS_MAX_LOG_LINES", defaultMaxLogLines), + MaxResourceSize: getEnvInt64("E2E_ARTIFACTS_MAX_RESOURCE_SIZE", defaultMaxResourceSize), + MaxTotalSize: getEnvInt64("E2E_ARTIFACTS_MAX_TOTAL_SIZE", defaultMaxTotalSize), + + CollectionTimeout: getEnvDuration("E2E_ARTIFACTS_TIMEOUT", defaultCollectionTimeout), + + NamespacePatterns: []string{"test-*"}, + PodLabelSelectors: []string{}, + } +} + +// Helper functions for ENV parsing + +func getEnvBool(key string, defaultValue bool) bool { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + result, err := strconv.ParseBool(value) + if err != nil { + return defaultValue + } + return result +} + +func getEnvInt(key string, defaultValue int) int { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + result, err := strconv.Atoi(value) + if err != nil { + return defaultValue + } + return result +} + +func getEnvInt64(key string, defaultValue int64) int64 { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + result, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return defaultValue + } + return result +} + +func getEnvString(key string, defaultValue string) string { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + return value +} + +func getEnvDuration(key string, defaultValue time.Duration) time.Duration { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + result, err := time.ParseDuration(value) + if err != nil { + return defaultValue + } + return result +} diff --git a/test/e2e/framework/artifacts/metadata.go b/test/e2e/framework/artifacts/metadata.go new file mode 100644 index 00000000..7eb03003 --- /dev/null +++ b/test/e2e/framework/artifacts/metadata.go @@ -0,0 +1,118 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package artifacts + +import ( + "encoding/json" + "fmt" + "time" +) + +// RunMetadata contains metadata about an entire test run +type RunMetadata struct { + RunID string `json:"run_id"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time,omitempty"` + TotalTests int `json:"total_tests"` + FailedTests int `json:"failed_tests"` + PassedTests int `json:"passed_tests"` + Environment map[string]string `json:"environment"` + ArtifactsDir string `json:"artifacts_dir"` + // Git information for tracking test run version + GitCommit string `json:"git_commit,omitempty"` + GitBranch string `json:"git_branch,omitempty"` + GitDirty string `json:"git_dirty,omitempty"` // "dirty", "staged", or empty if clean + Description string `json:"description,omitempty"` // Optional user description +} + +// TestMetadata contains metadata about a single test execution +type TestMetadata struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Duration time.Duration `json:"duration_ms"` // in milliseconds for JSON + Failed bool `json:"failed"` + FailureMessage string `json:"failure_message,omitempty"` + Labels []string `json:"labels"` + + // Test sequence tracking (for degradation analysis) + TestSequenceNumber int `json:"test_sequence_number"` // Which test in the run (1, 2, 3...) + OperatorAge time.Duration `json:"operator_age_seconds"` // How long operator has been running + + // Collected artifacts inventory + Artifacts ArtifactInventory `json:"artifacts"` +} + +// ArtifactInventory tracks what artifacts were collected +type ArtifactInventory struct { + PodCount int `json:"pod_count"` + LogFiles []string `json:"log_files"` + ResourceFiles []string `json:"resource_files"` + EventFiles []string `json:"event_files"` + TotalSizeBytes int64 `json:"total_size_bytes"` + CollectionTime string `json:"collection_time"` // Human-readable duration +} + +// MetadataBuilder helps build and write metadata files +type MetadataBuilder struct { + storage *Storage +} + +// NewMetadataBuilder creates a new metadata builder +func NewMetadataBuilder(storage *Storage) *MetadataBuilder { + return &MetadataBuilder{ + storage: storage, + } +} + +// WriteTestMetadata writes test metadata to JSON file +func (m *MetadataBuilder) WriteTestMetadata(meta TestMetadata, testDir string) error { + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal test metadata: %w", err) + } + + return m.storage.WriteFile(testDir, "", "metadata.json", data) +} + +// WriteRunMetadata writes run metadata to JSON file +func (m *MetadataBuilder) WriteRunMetadata(meta RunMetadata) error { + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal run metadata: %w", err) + } + + return m.storage.WriteFileInRunDir("metadata.json", data) +} + +// BuildTestMetadata creates TestMetadata from TestInfo +func BuildTestMetadata(info TestInfo, artifacts ArtifactInventory) TestMetadata { + return TestMetadata{ + Name: info.Name, + Namespace: info.Namespace, + StartTime: info.StartTime, + EndTime: info.EndTime, + Duration: info.Duration, + Failed: info.Failed, + FailureMessage: info.FailureMessage, + Labels: info.Labels, + TestSequenceNumber: info.SequenceNumber, + OperatorAge: info.OperatorAge, + Artifacts: artifacts, + } +} diff --git a/test/e2e/framework/artifacts/storage.go b/test/e2e/framework/artifacts/storage.go new file mode 100644 index 00000000..3ecca496 --- /dev/null +++ b/test/e2e/framework/artifacts/storage.go @@ -0,0 +1,327 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package artifacts + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "time" +) + +// Storage handles filesystem operations for artifact collection +type Storage struct { + baseDir string + runDir string + maxSize int64 + runID string +} + +// NewStorage creates a new storage instance with specified configuration +func NewStorage(baseDir string, runID string, maxSize int64) (*Storage, error) { + var runDir string + + // Check if baseDir already contains a run directory (e.g., from E2E_ARTIFACTS_DIR) + // This prevents nested run-{timestamp}/run-{timestamp}/ structure + if filepath.Base(baseDir) == "artifacts" && isRunDirectory(filepath.Dir(baseDir)) { + // baseDir is already inside a run directory (e.g., test/e2e/results/run-{timestamp}/artifacts/) + // Use it directly without creating another run-{runID} subdirectory + runDir = baseDir + } else { + // Standard case: create run-{runID} subdirectory + runDir = filepath.Join(baseDir, "run-"+runID) + } + + // Create run directory + if err := os.MkdirAll(runDir, 0755); err != nil { + return nil, fmt.Errorf("failed to create run directory %s: %w", runDir, err) + } + + return &Storage{ + baseDir: baseDir, + runDir: runDir, + maxSize: maxSize, + runID: runID, + }, nil +} + +// isRunDirectory checks if a directory name matches the run-{timestamp} pattern +func isRunDirectory(path string) bool { + base := filepath.Base(path) + return len(base) > 4 && base[:4] == "run-" +} + +// WriteFile writes content to a file within a test directory with size limits +// testDir: test-specific directory name (e.g., "test-normal-mode") +// category: subdirectory within test dir (e.g., "logs", "resources", "events") +// filename: name of the file to write +func (s *Storage) WriteFile(testDir, category, filename string, content []byte) error { + // Check and enforce size limit + if int64(len(content)) > s.maxSize { + content = s.truncateContent(content, "size limit exceeded") + } + + // Build full directory path + var dir string + if category != "" { + dir = filepath.Join(s.runDir, testDir, category) + } else { + dir = filepath.Join(s.runDir, testDir) + } + + // Create category directory + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", dir, err) + } + + // Write file atomically (write to temp, then rename) + path := filepath.Join(dir, filename) + tempPath := path + ".tmp" + + if err := os.WriteFile(tempPath, content, 0644); err != nil { + return fmt.Errorf("failed to write temp file %s: %w", tempPath, err) + } + + if err := os.Rename(tempPath, path); err != nil { + // Clean up temp file if rename fails + _ = os.Remove(tempPath) + return fmt.Errorf("failed to rename temp file %s to %s: %w", tempPath, path, err) + } + + return nil +} + +// WriteFileInRunDir writes a file directly in the run directory (not test-specific) +// Used for run-level metadata +func (s *Storage) WriteFileInRunDir(filename string, content []byte) error { + path := filepath.Join(s.runDir, filename) + tempPath := path + ".tmp" + + if err := os.WriteFile(tempPath, content, 0644); err != nil { + return fmt.Errorf("failed to write temp file %s: %w", tempPath, err) + } + + if err := os.Rename(tempPath, path); err != nil { + _ = os.Remove(tempPath) + return fmt.Errorf("failed to rename temp file %s to %s: %w", tempPath, path, err) + } + + return nil +} + +// WriteStream writes content from a reader to a file with size limits +// Useful for streaming command output without loading all into memory +func (s *Storage) WriteStream(testDir, category, filename string, reader io.Reader, maxLines int) error { + // Build full directory path + var dir string + if category != "" { + dir = filepath.Join(s.runDir, testDir, category) + } else { + dir = filepath.Join(s.runDir, testDir) + } + + // Create category directory + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("failed to create directory %s: %w", dir, err) + } + + // Write to temp file + path := filepath.Join(dir, filename) + tempPath := path + ".tmp" + + tempFile, err := os.Create(tempPath) + if err != nil { + return fmt.Errorf("failed to create temp file %s: %w", tempPath, err) + } + defer tempFile.Close() + + // Copy with size limit + written, err := io.CopyN(tempFile, reader, s.maxSize) + if err != nil && err != io.EOF { + // If we hit the limit, add truncation marker + if written >= s.maxSize { + truncationMarker := []byte("\n\n... [TRUNCATED - exceeds size limit] ...\n") + _, _ = tempFile.Write(truncationMarker) + } + } + + tempFile.Close() + + // Rename to final path + if err := os.Rename(tempPath, path); err != nil { + _ = os.Remove(tempPath) + return fmt.Errorf("failed to rename temp file %s to %s: %w", tempPath, path, err) + } + + return nil +} + +// GetRunDir returns the run directory path +func (s *Storage) GetRunDir() string { + return s.runDir +} + +// GetRunID returns the run ID +func (s *Storage) GetRunID() string { + return s.runID +} + +// truncateContent truncates content to fit within maxSize and adds a marker +func (s *Storage) truncateContent(content []byte, reason string) []byte { + marker := []byte(fmt.Sprintf("\n\n... [TRUNCATED: %s - max %d bytes] ...\n", reason, s.maxSize)) + + // If marker itself is too large, truncate it + if int64(len(marker)) >= s.maxSize { + return marker[:s.maxSize] + } + + // Calculate how much content we can keep + keepSize := s.maxSize - int64(len(marker)) + if keepSize < 0 { + keepSize = 0 + } + + // Keep the end of the content (most recent logs are usually most relevant) + // But also include first few bytes to show what file it is + headerSize := int64(100) + if headerSize > keepSize/2 { + headerSize = keepSize / 2 + } + + var truncated []byte + if headerSize > 0 && int64(len(content)) > headerSize { + // Include header + marker + tail + tailSize := keepSize - headerSize + tailStart := int64(len(content)) - tailSize + if tailStart < headerSize { + tailStart = headerSize + } + + truncated = append(truncated, content[:headerSize]...) + truncated = append(truncated, []byte("\n... [CONTENT SKIPPED] ...\n")...) + if tailStart < int64(len(content)) { + truncated = append(truncated, content[tailStart:]...) + } + } else { + // Just take what fits + truncated = content[:keepSize] + } + + return append(truncated, marker...) +} + +// TruncateLogLines truncates log output to specified number of lines +// Takes the LAST N lines (most recent logs are most relevant for debugging) +func TruncateLogLines(content []byte, maxLines int) []byte { + if maxLines <= 0 { + return content + } + + lines := []byte{} + lineCount := 0 + newlineCount := 0 + + // Count newlines from the end + for i := len(content) - 1; i >= 0; i-- { + if content[i] == '\n' { + newlineCount++ + if newlineCount >= maxLines { + // Found enough lines, this is our cut point + lines = content[i+1:] + lineCount = maxLines + break + } + } + } + + // If we didn't find enough newlines, return all content + if lineCount == 0 { + return content + } + + // Skip leading empty lines and trim leading whitespace from first line + start := 0 + for start < len(lines) { + // Find end of current line + end := start + for end < len(lines) && lines[end] != '\n' { + end++ + } + + // Check if line has any non-whitespace content + lineContent := bytes.TrimSpace(lines[start:end]) + if len(lineContent) > 0 { + // Found first non-empty line + // Build result: trimmed first line + rest + result := lineContent + if end < len(lines) { + // Append the rest (from \n onwards) + result = append(result, lines[end:]...) + } + lines = result + break + } + + // Move to next line (skip the \n) + start = end + 1 + } + + // Add truncation marker at the beginning + marker := []byte(fmt.Sprintf("... [Showing last %d lines] ...\n", lineCount)) + return append(marker, lines...) +} + +// CreateTestDir creates a directory for a specific test +func (s *Storage) CreateTestDir(testName string) (string, error) { + // Sanitize test name for filesystem + sanitized := sanitizeFilename(testName) + testDir := filepath.Join(s.runDir, sanitized) + + if err := os.MkdirAll(testDir, 0755); err != nil { + return "", fmt.Errorf("failed to create test directory %s: %w", testDir, err) + } + + return sanitized, nil +} + +// sanitizeFilename removes characters that are problematic in filenames +func sanitizeFilename(name string) string { + // Replace spaces and problematic characters with hyphens + result := []byte(name) + for i, c := range result { + switch c { + case '/', '\\', ':', '*', '?', '"', '<', '>', '|', ' ': + result[i] = '-' + } + } + + // Limit length to avoid filesystem issues + const maxLength = 200 + if len(result) > maxLength { + // Use a timestamp suffix to ensure uniqueness + suffix := fmt.Sprintf("-%d", time.Now().Unix()) + cutPoint := maxLength - len(suffix) + if cutPoint < 0 { + cutPoint = 0 + } + result = append(result[:cutPoint], []byte(suffix)...) + } + + return string(result) +} diff --git a/test/e2e/framework/artifacts/storage_test.go b/test/e2e/framework/artifacts/storage_test.go new file mode 100644 index 00000000..3af92c63 --- /dev/null +++ b/test/e2e/framework/artifacts/storage_test.go @@ -0,0 +1,55 @@ +package artifacts + +import ( + "bytes" + "testing" +) + +func TestTruncateLogLines_RemovesLeadingWhitespace(t *testing.T) { + tests := []struct { + name string + input string + maxLines int + want string + }{ + { + name: "removes leading newlines and spaces", + input: "\n\n \t2025-11-14T19:58:40Z\tINFO\tstart Reconcile\nline2\nline3", + maxLines: 3, + want: "... [Showing last 3 lines] ...\n2025-11-14T19:58:40Z\tINFO\tstart Reconcile\nline2\nline3", + }, + { + name: "handles logs without leading whitespace", + input: "line1\nline2\nline3\nline4\nline5", + maxLines: 3, + want: "... [Showing last 3 lines] ...\nline3\nline4\nline5", + }, + { + name: "keeps content when less than maxLines", + input: "line1\nline2", + maxLines: 5, + want: "line1\nline2", + }, + { + name: "trims leading whitespace from first line but preserves it in subsequent lines", + input: "line1\nline2\nline3 with content\n indented line4\n indented line5", + maxLines: 3, + want: "... [Showing last 3 lines] ...\nline3 with content\n indented line4\n indented line5", + }, + { + name: "handles real operator log format", + input: "line1\nline2\nline3\nline4\n2025-11-14T19:58:40Z\tINFO\tstart Reconcile", + maxLines: 1, + want: "... [Showing last 1 lines] ...\n2025-11-14T19:58:40Z\tINFO\tstart Reconcile", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := TruncateLogLines([]byte(tt.input), tt.maxLines) + if !bytes.Equal(got, []byte(tt.want)) { + t.Errorf("TruncateLogLines() = %q, want %q", string(got), tt.want) + } + }) + } +} diff --git a/test/e2e/framework/assertions/matchers.go b/test/e2e/framework/assertions/matchers.go new file mode 100644 index 00000000..227721d4 --- /dev/null +++ b/test/e2e/framework/assertions/matchers.go @@ -0,0 +1,300 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package assertions + +import ( + "fmt" + "strings" + + "github.com/onsi/gomega/types" + + "github.com/kaasops/vector-operator/test/e2e/framework/kubectl" +) + +// PipelineResource represents a pipeline for matching +type PipelineResource struct { + namespace string + name string + kubectl *kubectl.Client +} + +// NewPipelineResource creates a new pipeline resource wrapper +func NewPipelineResource(namespace, name string) *PipelineResource { + return &PipelineResource{ + namespace: namespace, + name: name, + kubectl: kubectl.NewClient(namespace), + } +} + +// resourceType returns the correct resource type based on namespace +// Empty namespace = cluster-scoped (ClusterVectorPipeline) +// Non-empty namespace = namespaced (VectorPipeline) +func (p *PipelineResource) resourceType() string { + if p.namespace == "" { + return "clustervectorpipeline" + } + return "vectorpipeline" +} + +// BeValid matcher for pipeline validity +type beValidMatcher struct{} + +func (m *beValidMatcher) Match(actual interface{}) (success bool, err error) { + pipeline, ok := actual.(*PipelineResource) + if !ok { + return false, fmt.Errorf("BeValid matcher expects a *PipelineResource") + } + + result, err := pipeline.kubectl.GetWithJsonPath(pipeline.resourceType(), pipeline.name, ".status.configCheckResult") + if err != nil { + return false, err + } + + return result == "true", nil +} + +func (m *beValidMatcher) FailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s to be valid", pipeline.namespace, pipeline.name) +} + +func (m *beValidMatcher) NegatedFailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s not to be valid", pipeline.namespace, pipeline.name) +} + +// BeValid returns a matcher that checks if a pipeline is valid +func BeValid() types.GomegaMatcher { + return &beValidMatcher{} +} + +// HaveSplitModeEnabled matcher +type haveSplitModeEnabledMatcher struct{} + +func (m *haveSplitModeEnabledMatcher) Match(actual interface{}) (success bool, err error) { + pipeline, ok := actual.(*PipelineResource) + if !ok { + return false, fmt.Errorf("HaveSplitModeEnabled matcher expects a *PipelineResource") + } + + result, err := pipeline.kubectl.GetWithJsonPath(pipeline.resourceType(), pipeline.name, ".status.splitMode.enabled") + if err != nil { + return false, err + } + + return result == "true", nil +} + +func (m *haveSplitModeEnabledMatcher) FailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s to have split mode enabled", pipeline.namespace, pipeline.name) +} + +func (m *haveSplitModeEnabledMatcher) NegatedFailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s not to have split mode enabled", pipeline.namespace, pipeline.name) +} + +// HaveSplitModeEnabled returns a matcher that checks if split mode is enabled +func HaveSplitModeEnabled() types.GomegaMatcher { + return &haveSplitModeEnabledMatcher{} +} + +// HaveRole matcher +type haveRoleMatcher struct { + expectedRole string +} + +func (m *haveRoleMatcher) Match(actual interface{}) (success bool, err error) { + pipeline, ok := actual.(*PipelineResource) + if !ok { + return false, fmt.Errorf("HaveRole matcher expects a *PipelineResource") + } + + result, err := pipeline.kubectl.GetWithJsonPath(pipeline.resourceType(), pipeline.name, ".status.role") + if err != nil { + return false, err + } + + return result == m.expectedRole, nil +} + +func (m *haveRoleMatcher) FailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s to have role %s", pipeline.namespace, pipeline.name, m.expectedRole) +} + +func (m *haveRoleMatcher) NegatedFailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s not to have role %s", pipeline.namespace, pipeline.name, m.expectedRole) +} + +// HaveRole returns a matcher that checks the pipeline role +func HaveRole(role string) types.GomegaMatcher { + return &haveRoleMatcher{expectedRole: role} +} + +// ServiceResource represents a service for matching +type ServiceResource struct { + namespace string + name string + kubectl *kubectl.Client +} + +// NewServiceResource creates a new service resource wrapper +func NewServiceResource(namespace, name string) *ServiceResource { + return &ServiceResource{ + namespace: namespace, + name: name, + kubectl: kubectl.NewClient(namespace), + } +} + +// Exist matcher for service existence +type existMatcher struct{} + +func (m *existMatcher) Match(actual interface{}) (success bool, err error) { + service, ok := actual.(*ServiceResource) + if !ok { + return false, fmt.Errorf("Exist matcher expects a *ServiceResource") + } + + _, err = service.kubectl.Get("service", service.name) + return err == nil, nil +} + +func (m *existMatcher) FailureMessage(actual interface{}) string { + service := actual.(*ServiceResource) + return fmt.Sprintf("Expected service %s/%s to exist", service.namespace, service.name) +} + +func (m *existMatcher) NegatedFailureMessage(actual interface{}) string { + service := actual.(*ServiceResource) + return fmt.Sprintf("Expected service %s/%s not to exist", service.namespace, service.name) +} + +// Exist returns a matcher that checks if a service exists +func Exist() types.GomegaMatcher { + return &existMatcher{} +} + +// HavePort matcher +type havePortMatcher struct { + expectedPort string +} + +func (m *havePortMatcher) Match(actual interface{}) (success bool, err error) { + service, ok := actual.(*ServiceResource) + if !ok { + return false, fmt.Errorf("HavePort matcher expects a *ServiceResource") + } + + port, err := service.kubectl.GetWithJsonPath("service", service.name, ".spec.ports[0].port") + if err != nil { + return false, err + } + + return port == m.expectedPort, nil +} + +func (m *havePortMatcher) FailureMessage(actual interface{}) string { + service := actual.(*ServiceResource) + return fmt.Sprintf("Expected service %s/%s to have port %s", service.namespace, service.name, m.expectedPort) +} + +func (m *havePortMatcher) NegatedFailureMessage(actual interface{}) string { + service := actual.(*ServiceResource) + return fmt.Sprintf("Expected service %s/%s not to have port %s", service.namespace, service.name, m.expectedPort) +} + +// HavePort returns a matcher that checks the service port +func HavePort(port string) types.GomegaMatcher { + return &havePortMatcher{expectedPort: port} +} + +// BeInvalid matcher for pipeline invalidity +type beInvalidMatcher struct{} + +func (m *beInvalidMatcher) Match(actual interface{}) (success bool, err error) { + pipeline, ok := actual.(*PipelineResource) + if !ok { + return false, fmt.Errorf("BeInvalid matcher expects a *PipelineResource") + } + + result, err := pipeline.kubectl.GetWithJsonPath(pipeline.resourceType(), pipeline.name, ".status.configCheckResult") + if err != nil { + return false, err + } + + return result == "false", nil +} + +func (m *beInvalidMatcher) FailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s to be invalid", pipeline.namespace, pipeline.name) +} + +func (m *beInvalidMatcher) NegatedFailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s not to be invalid", pipeline.namespace, pipeline.name) +} + +// BeInvalid returns a matcher that checks if a pipeline is invalid +func BeInvalid() types.GomegaMatcher { + return &beInvalidMatcher{} +} + +// HaveErrorContaining matcher for error messages +type haveErrorContainingMatcher struct { + expectedSubstring string +} + +func (m *haveErrorContainingMatcher) Match(actual interface{}) (success bool, err error) { + pipeline, ok := actual.(*PipelineResource) + if !ok { + return false, fmt.Errorf("HaveErrorContaining matcher expects a *PipelineResource") + } + + reason, err := pipeline.kubectl.GetWithJsonPath(pipeline.resourceType(), pipeline.name, ".status.reason") + if err != nil { + return false, err + } + + // Simple substring check (case-insensitive) + lowerReason := strings.ToLower(reason) + lowerExpected := strings.ToLower(m.expectedSubstring) + + return strings.Contains(lowerReason, lowerExpected), nil +} + +func (m *haveErrorContainingMatcher) FailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s to have error containing '%s'", + pipeline.namespace, pipeline.name, m.expectedSubstring) +} + +func (m *haveErrorContainingMatcher) NegatedFailureMessage(actual interface{}) string { + pipeline := actual.(*PipelineResource) + return fmt.Sprintf("Expected pipeline %s/%s not to have error containing '%s'", + pipeline.namespace, pipeline.name, m.expectedSubstring) +} + +// HaveErrorContaining returns a matcher that checks if error message contains substring +func HaveErrorContaining(substring string) types.GomegaMatcher { + return &haveErrorContainingMatcher{expectedSubstring: substring} +} diff --git a/test/e2e/framework/config/constants.go b/test/e2e/framework/config/constants.go new file mode 100644 index 00000000..8bde68e6 --- /dev/null +++ b/test/e2e/framework/config/constants.go @@ -0,0 +1,50 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +// Test labels for selective test execution +const ( + // Execution speed labels + LabelSmoke = "smoke" + LabelFast = "fast" + LabelSlow = "slow" + LabelRegression = "regression" + LabelStress = "stress" + LabelParallel = "parallel" + + // Priority labels (P0 = critical, must always pass) + LabelP0 = "p0" + LabelP1 = "p1" + LabelP2 = "p2" + + // Category labels + LabelSecurity = "security" + LabelConstraint = "constraint" +) + +// Resource naming suffixes +const ( + AggregatorSuffix = "-aggregator" + AgentSuffix = "-agent" +) + +// Kubernetes labels +const ( + ComponentLabel = "app.kubernetes.io/component" + AggregatorComponent = "Aggregator" + AgentComponent = "Agent" +) diff --git a/test/e2e/framework/config/timeouts.go b/test/e2e/framework/config/timeouts.go new file mode 100644 index 00000000..82a0fe7f --- /dev/null +++ b/test/e2e/framework/config/timeouts.go @@ -0,0 +1,92 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "os" + "time" +) + +// Default timeout values +const ( + // Resource creation timeouts + defaultDeploymentCreateTimeout = 120 * time.Second // Increased for resource-heavy aggregator deployments + defaultDeploymentReadyTimeout = 120 * time.Second + defaultNamespaceDeleteTimeout = 120 * time.Second // Increased timeout to handle slow namespace termination + + // Pipeline validation timeouts + defaultPipelineValidTimeout = 2 * time.Minute + defaultConfigCheckTimeout = 30 * time.Second + + // Service check timeouts + defaultServiceCreateTimeout = 2 * time.Minute + + // Polling intervals + defaultDefaultPollInterval = 2 * time.Second + defaultFastPollInterval = 1 * time.Second + defaultSlowPollInterval = 2 * time.Second // Reduced from 5s - more responsive polling + + // Test spec timeouts + defaultDefaultTestTimeout = 5 * time.Minute + defaultLongTestTimeout = 10 * time.Minute +) + +// Configurable timeout variables (can be overridden via environment variables) +var ( + // Resource creation timeouts + DeploymentCreateTimeout = getEnvDuration("E2E_DEPLOYMENT_CREATE_TIMEOUT", defaultDeploymentCreateTimeout) + DeploymentReadyTimeout = getEnvDuration("E2E_DEPLOYMENT_READY_TIMEOUT", defaultDeploymentReadyTimeout) + NamespaceDeleteTimeout = getEnvDuration("E2E_NAMESPACE_DELETE_TIMEOUT", defaultNamespaceDeleteTimeout) + + // Pipeline validation timeouts + PipelineValidTimeout = getEnvDuration("E2E_PIPELINE_VALID_TIMEOUT", defaultPipelineValidTimeout) + ConfigCheckTimeout = getEnvDuration("E2E_CONFIG_CHECK_TIMEOUT", defaultConfigCheckTimeout) + + // Service check timeouts + ServiceCreateTimeout = getEnvDuration("E2E_SERVICE_CREATE_TIMEOUT", defaultServiceCreateTimeout) + + // Polling intervals + DefaultPollInterval = getEnvDuration("E2E_DEFAULT_POLL_INTERVAL", defaultDefaultPollInterval) + FastPollInterval = getEnvDuration("E2E_FAST_POLL_INTERVAL", defaultFastPollInterval) + SlowPollInterval = getEnvDuration("E2E_SLOW_POLL_INTERVAL", defaultSlowPollInterval) + + // Test spec timeouts + DefaultTestTimeout = getEnvDuration("E2E_DEFAULT_TEST_TIMEOUT", defaultDefaultTestTimeout) + LongTestTimeout = getEnvDuration("E2E_LONG_TEST_TIMEOUT", defaultLongTestTimeout) +) + +// getEnvDuration reads a duration from environment variable, falling back to default if not set or invalid +func getEnvDuration(envVar string, defaultValue time.Duration) time.Duration { + if val := os.Getenv(envVar); val != "" { + if duration, err := time.ParseDuration(val); err == nil { + return duration + } + // If parsing fails, fall back to default (silently to avoid test noise) + } + return defaultValue +} + +// GetPollInterval returns appropriate poll interval based on timeout +func GetPollInterval(timeout time.Duration) time.Duration { + if timeout < 30*time.Second { + return FastPollInterval + } + if timeout > 2*time.Minute { + return SlowPollInterval + } + return DefaultPollInterval +} diff --git a/test/e2e/framework/errors/errors.go b/test/e2e/framework/errors/errors.go new file mode 100644 index 00000000..c2bd06a0 --- /dev/null +++ b/test/e2e/framework/errors/errors.go @@ -0,0 +1,100 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package errors + +import ( + "strings" +) + +// Centralized error classification for e2e tests +// Provides consistent error handling across kubectl operations + +// IsAlreadyExists checks if error indicates resource already exists +func IsAlreadyExists(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "AlreadyExists") || + strings.Contains(errStr, "already exists") +} + +// IsNotFound checks if error indicates resource not found +func IsNotFound(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "NotFound") || + strings.Contains(errStr, "not found") || + strings.Contains(errStr, "(NotFound)") +} + +// IsConflict checks if error indicates resource conflict +func IsConflict(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "Conflict") || + strings.Contains(errStr, "the object has been modified") +} + +// IsTimeout checks if error indicates timeout +func IsTimeout(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "timeout") || + strings.Contains(errStr, "timed out") || + strings.Contains(errStr, "context deadline exceeded") +} + +// IsConnectionError checks if error indicates connection/network issue +func IsConnectionError(err error) bool { + if err == nil { + return false + } + errStr := err.Error() + return strings.Contains(errStr, "connection refused") || + strings.Contains(errStr, "i/o timeout") || + strings.Contains(errStr, "network") || + strings.Contains(errStr, "dial tcp") +} + +// IsTransient checks if error is likely transient and retriable +func IsTransient(err error) bool { + if err == nil { + return false + } + return IsTimeout(err) || + IsConnectionError(err) || + IsConflict(err) || + strings.Contains(err.Error(), "Internal error") || + strings.Contains(err.Error(), "TooManyRequests") || + strings.Contains(err.Error(), "ServerTimeout") +} + +// IsIgnorable checks if error can be safely ignored in test setup/teardown +func IsIgnorable(err error) bool { + if err == nil { + return true + } + // AlreadyExists and NotFound are often acceptable in test lifecycle + return IsAlreadyExists(err) || IsNotFound(err) +} diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go new file mode 100644 index 00000000..32ba098d --- /dev/null +++ b/test/e2e/framework/framework.go @@ -0,0 +1,1154 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2/types" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/kaasops/vector-operator/test/e2e/framework/config" + "github.com/kaasops/vector-operator/test/e2e/framework/errors" + "github.com/kaasops/vector-operator/test/e2e/framework/kubectl" + "github.com/kaasops/vector-operator/test/e2e/framework/recorder" +) + +const ( + // MaxConfigSize is the maximum allowed size for base64-encoded config data (10MB) + // This prevents DoS attacks via extremely large config payloads + MaxConfigSize = 10 * 1024 * 1024 // 10MB +) + +// Framework provides a high-level API for e2e tests +type Framework struct { + namespace string + kubectl *kubectl.Client + isShared bool + metrics *TestMetrics + recorder *recorder.TestRecorder + dryRun bool + recordSteps bool + TestDataPath string // Path to test data directory (configurable via E2E_TESTDATA_PATH) +} + +// TestMetrics tracks timing information for test operations +type TestMetrics struct { + SetupTime time.Duration + DeploymentWaitTime time.Duration + PipelineValidationTime time.Duration + CleanupTime time.Duration +} + +// Global shared framework instance +var sharedFramework *Framework + +// frameworkRegistry stores framework instances for artifact collection +// Key: namespace, Value: *Framework +// DEPRECATED: This is being phased out in favor of Ginkgo report entries. +// New code should use AddReportEntry() in Setup() and retrieve from ReportAfterEach. +var frameworkRegistry sync.Map + +// FrameworkContextKey is the key type for storing Framework in context +// Using a custom type prevents collisions with other context keys +type FrameworkContextKey struct{} + +// frameworkReportEntryName is the name used when storing framework in Ginkgo report entries +const frameworkReportEntryName = "framework-instance" + +// NewFramework creates a new isolated test framework with its own namespace +func NewFramework(namespace string) *Framework { + // Get test data path from environment or use default + testDataPath := os.Getenv("E2E_TESTDATA_PATH") + if testDataPath == "" { + testDataPath = filepath.Join("test", "e2e", "testdata") + } + + f := &Framework{ + namespace: namespace, + kubectl: kubectl.NewClient(namespace), + isShared: false, + metrics: &TestMetrics{}, + TestDataPath: testDataPath, + } + + // Check for dry-run or recording mode + if os.Getenv("E2E_DRY_RUN") == "true" { + f.dryRun = true + f.recorder = recorder.NewTestRecorder(namespace) + f.recordSteps = true + } else if os.Getenv("E2E_RECORD_STEPS") == "true" { + f.recordSteps = true + f.recorder = recorder.NewTestRecorder(namespace) + } + + return f +} + +// NewUniqueFramework creates a new framework with a unique timestamped namespace +// This prevents namespace collisions when tests run in parallel or when cleanup is slow +func NewUniqueFramework(baseName string) *Framework { + // Use nanosecond timestamp + counter for uniqueness + timestamp := time.Now().UnixNano() + uniqueNS := fmt.Sprintf("%s-%d", baseName, timestamp) + + // Get test data path from environment or use default + testDataPath := os.Getenv("E2E_TESTDATA_PATH") + if testDataPath == "" { + testDataPath = filepath.Join("test", "e2e", "testdata") + } + + f := &Framework{ + namespace: uniqueNS, + kubectl: kubectl.NewClient(uniqueNS), + isShared: false, + metrics: &TestMetrics{}, + TestDataPath: testDataPath, + } + + // Check for dry-run or recording mode + if os.Getenv("E2E_DRY_RUN") == "true" { + f.dryRun = true + f.recorder = recorder.NewTestRecorder(uniqueNS) + f.recordSteps = true + } else if os.Getenv("E2E_RECORD_STEPS") == "true" { + f.recordSteps = true + f.recorder = recorder.NewTestRecorder(uniqueNS) + } + + return f +} + +// Shared returns a shared framework instance that reuses the same namespace +// This is useful for parallel tests that don't interfere with each other +func Shared(namespace string) *Framework { + if sharedFramework == nil { + // Get test data path from environment or use default + testDataPath := os.Getenv("E2E_TESTDATA_PATH") + if testDataPath == "" { + testDataPath = filepath.Join("test", "e2e", "testdata") + } + + sharedFramework = &Framework{ + namespace: namespace, + kubectl: kubectl.NewClient(namespace), + isShared: true, + metrics: &TestMetrics{}, + TestDataPath: testDataPath, + } + + // Check for dry-run or recording mode + if os.Getenv("E2E_DRY_RUN") == "true" { + sharedFramework.dryRun = true + sharedFramework.recorder = recorder.NewTestRecorder(namespace) + sharedFramework.recordSteps = true + } else if os.Getenv("E2E_RECORD_STEPS") == "true" { + sharedFramework.recordSteps = true + sharedFramework.recorder = recorder.NewTestRecorder(namespace) + } + } + return sharedFramework +} + +// Setup performs the test environment setup +func (f *Framework) Setup() { + // Store framework in Ginkgo report entries for artifact collection + // This is the preferred method as it directly associates framework with the current test + // and works correctly with parallel test execution + AddReportEntry(frameworkReportEntryName, f) + + // DEPRECATED: Also store in global registry for backward compatibility + // This will be removed in a future version once all code migrates to using report entries + frameworkRegistry.Store(f.namespace, f) + + start := time.Now() + defer func() { + f.metrics.SetupTime = time.Since(start) + }() + + By(fmt.Sprintf("creating test namespace: %s", f.namespace)) + err := kubectl.CreateNamespace(f.namespace) + if err != nil { + // Check if it's an ignorable error (AlreadyExists, NotFound) + if errors.IsIgnorable(err) { + GinkgoWriter.Printf("Warning: namespace creation failed (might already exist): %v\n", err) + } else { + // Only fail for non-ignorable errors + Expect(err).NotTo(HaveOccurred()) + } + } + + // Wait for namespace to be ready before proceeding + By(fmt.Sprintf("waiting for namespace to be ready: %s", f.namespace)) + Eventually(func() bool { + ns, err := kubectl.GetNamespace(f.namespace) + if err != nil { + GinkgoWriter.Printf("Failed to get namespace status: %v\n", err) + return false + } + // Check namespace is Active (not Terminating) + return ns.Status.Phase == "Active" + }, config.DeploymentReadyTimeout, config.DefaultPollInterval).Should(BeTrue(), + fmt.Sprintf("namespace %s should be Active", f.namespace)) +} + +// Teardown performs the test environment cleanup +func (f *Framework) Teardown() { + // Export test plan if recording is enabled + if f.recorder != nil && f.recordSteps { + f.ExportTestPlan() + } + + // Don't cleanup shared namespaces immediately + if f.isShared { + return + } + + start := time.Now() + defer func() { + f.metrics.CleanupTime = time.Since(start) + }() + + By(fmt.Sprintf("cleaning up test namespace: %s", f.namespace)) + err := kubectl.DeleteNamespace(f.namespace, fmt.Sprintf("%ds", int(config.NamespaceDeleteTimeout.Seconds()))) + if err != nil { + GinkgoWriter.Printf("Warning: namespace cleanup failed: %v\n", err) + } + + // NOTE: Do NOT delete from frameworkRegistry here! + // ReportAfterEach runs AFTER AfterAll/Teardown, and needs the framework + // for artifact collection. The registry will be cleaned up when the process exits. + // frameworkRegistry.Delete(f.namespace) +} + +// Namespace returns the test namespace +func (f *Framework) Namespace() string { + return f.namespace +} + +// ApplyTestData loads and applies a test manifest from testdata directory +// It automatically replaces any hardcoded namespace with the framework's namespace +func (f *Framework) ApplyTestData(path string) { + By(fmt.Sprintf("applying test data: %s", path)) + + content, err := os.ReadFile(filepath.Join(f.TestDataPath, path)) + Expect(err).NotTo(HaveOccurred(), "Failed to load test data from %s", path) + + // Replace namespace in YAML if present + yamlContent := replaceNamespace(string(content), f.namespace) + + err = f.kubectl.Apply(yamlContent) + Expect(err).NotTo(HaveOccurred(), "Failed to apply test data %s in namespace %s", path, f.namespace) +} + +// ApplyTestDataWithoutNamespaceReplacement loads and applies a test manifest WITHOUT namespace replacement +// Use this when you need to apply resources to specific namespaces +func (f *Framework) ApplyTestDataWithoutNamespaceReplacement(path string) { + By(fmt.Sprintf("applying test data without namespace replacement: %s", path)) + + content, err := os.ReadFile(filepath.Join(f.TestDataPath, path)) + Expect(err).NotTo(HaveOccurred(), "Failed to load test data from %s", path) + + // Apply without forcing namespace (YAML contains the correct namespace) + err = f.kubectl.ApplyWithoutNamespaceOverride(string(content)) + Expect(err).NotTo(HaveOccurred(), "Failed to apply test data %s", path) +} + +// replaceNamespace replaces hardcoded namespaces in YAML content +func replaceNamespace(yaml, namespace string) string { + // This is a simple replacement - for production use, proper YAML parsing might be better + // But for tests this is sufficient + lines := []string{} + for _, line := range splitLines(yaml) { + // Replace namespace: with namespace: + if len(line) > 12 && line[:12] == " namespace:" { + lines = append(lines, fmt.Sprintf(" namespace: %s", namespace)) + } else { + lines = append(lines, line) + } + } + return joinLines(lines) +} + +// splitLines splits string by newlines +func splitLines(s string) []string { + return strings.Split(s, "\n") +} + +// joinLines joins lines with newlines +func joinLines(lines []string) string { + return strings.Join(lines, "\n") +} + +// ApplyYAML applies raw YAML content +func (f *Framework) ApplyYAML(yamlContent string) { + err := f.kubectl.Apply(yamlContent) + Expect(err).NotTo(HaveOccurred(), "Failed to apply YAML in namespace %s", f.namespace) +} + +// WaitForDeploymentReady waits for a deployment to be ready +func (f *Framework) WaitForDeploymentReady(name string) { + By(fmt.Sprintf("waiting for deployment %s to be ready", name)) + start := time.Now() + defer func() { + duration := time.Since(start) + f.metrics.DeploymentWaitTime += duration + GinkgoWriter.Printf("⏱️ Deployment %s ready in %v\n", name, duration) + }() + + f.kubectl.WaitForDeploymentReady(name) +} + +// WaitForPipelineValid waits for a pipeline to become valid +func (f *Framework) WaitForPipelineValid(name string) { + By(fmt.Sprintf("waiting for pipeline %s to become valid", name)) + start := time.Now() + defer func() { + duration := time.Since(start) + f.metrics.PipelineValidationTime += duration + GinkgoWriter.Printf("⏱️ Pipeline %s validated in %v\n", name, duration) + }() + + f.kubectl.WaitForPipelineValid(name) +} + +// WaitForPipelineInvalid waits for a pipeline to become invalid (for negative tests) +func (f *Framework) WaitForPipelineInvalid(name string) { + By(fmt.Sprintf("waiting for pipeline %s to become invalid", name)) + f.kubectl.WaitForPipelineInvalid(name) +} + +// GetPipelineStatus retrieves a specific status field from a pipeline +func (f *Framework) GetPipelineStatus(name string, field string) string { + result, err := f.kubectl.GetWithJsonPath("vectorpipeline", name, fmt.Sprintf(".status.%s", field)) + Expect(err).NotTo(HaveOccurred(), + "Failed to get pipeline %s status field %s in namespace %s", name, field, f.namespace) + return result +} + +// GetServicePort retrieves the port of a service +func (f *Framework) GetServicePort(name string) string { + result, err := f.kubectl.GetWithJsonPath("service", name, ".spec.ports[0].port") + Expect(err).NotTo(HaveOccurred(), + "Failed to get service %s port in namespace %s", name, f.namespace) + return result +} + +// TryGetServicePort retrieves the port of a service without failing if not found +func (f *Framework) TryGetServicePort(name string) (string, error) { + return f.kubectl.GetWithJsonPath("service", name, ".spec.ports[0].port") +} + +// CreateMultiplePipelinesFromTemplate creates N pipelines from a template by replacing a placeholder +func (f *Framework) CreateMultiplePipelinesFromTemplate(templatePath, placeholder string, count int) time.Duration { + start := time.Now() + + content, err := os.ReadFile(filepath.Join(f.TestDataPath, templatePath)) + Expect(err).NotTo(HaveOccurred(), "Failed to load template from %s", templatePath) + + template := string(content) + + for i := 1; i <= count; i++ { + pipelineName := fmt.Sprintf("pipeline-%03d", i) + yaml := replaceNamespace(template, f.namespace) + yaml = replacePlaceholder(yaml, placeholder, pipelineName) + + err = f.kubectl.Apply(yaml) + Expect(err).NotTo(HaveOccurred(), + "Failed to apply pipeline %s from template %s in namespace %s", pipelineName, templatePath, f.namespace) + } + + return time.Since(start) +} + +// replacePlaceholder replaces a placeholder in YAML content +func replacePlaceholder(yaml, placeholder, value string) string { + return strings.ReplaceAll(yaml, placeholder, value) +} + +// CountValidPipelines counts how many pipelines are valid in the namespace +func (f *Framework) CountValidPipelines() (int, error) { + result, err := f.kubectl.GetWithJsonPath("vectorpipeline", "", ".items[*].status.configCheckResult") + if err != nil { + return 0, err + } + + if result == "" { + return 0, nil + } + + validCount := 0 + for _, status := range splitFields(result) { + if status == "true" { + validCount++ + } + } + + return validCount, nil +} + +// CountPipelines returns the total number of pipelines in the namespace +func (f *Framework) CountPipelines() (int, error) { + result, err := f.kubectl.GetAll("vectorpipeline", "") + if err != nil { + return 0, err + } + + if result == "" { + return 0, nil + } + + return len(splitFields(result)), nil +} + +// CountServicesContaining counts services whose name contains the given substring +func (f *Framework) CountServicesContaining(substring string) (int, error) { + result, err := f.kubectl.GetAll("service", "") + if err != nil { + return 0, err + } + + if result == "" { + return 0, nil + } + + count := 0 + for _, svc := range splitFields(result) { + if svc != "" && containsSubstring(svc, substring) { + count++ + } + } + + return count, nil +} + +// containsSubstring checks if a string contains a substring +func containsSubstring(s, substr string) bool { + return strings.Contains(s, substr) +} + +// ExpectServiceExists verifies that a service exists +func (f *Framework) ExpectServiceExists(name string) { + By(fmt.Sprintf("verifying service %s exists", name)) + _, err := f.kubectl.Get("service", name) + Expect(err).NotTo(HaveOccurred(), + "Expected service %s to exist in namespace %s", name, f.namespace) +} + +// CountServicesWithLabel counts services matching a label selector +func (f *Framework) CountServicesWithLabel(labelSelector string) int { + result, err := f.kubectl.GetAll("service", labelSelector) + Expect(err).NotTo(HaveOccurred(), + "Failed to get services with label %s in namespace %s", labelSelector, f.namespace) + + if result == "" { + return 0 + } + + count := 0 + for _, svc := range splitFields(result) { + if svc != "" { + count++ + } + } + return count +} + +// WaitForServiceCount waits for a specific number of services +func (f *Framework) WaitForServiceCount(labelSelector string, expectedCount int, timeout time.Duration) { + By(fmt.Sprintf("waiting for %d services with label %s", expectedCount, labelSelector)) + f.kubectl.WaitForServiceCount(labelSelector, expectedCount, timeout) +} + +// PrintMetrics prints timing metrics for the test +func (f *Framework) PrintMetrics() { + GinkgoWriter.Println("\n📊 Test Metrics:") + GinkgoWriter.Printf(" Setup: %v\n", f.metrics.SetupTime) + GinkgoWriter.Printf(" Deployment Wait: %v\n", f.metrics.DeploymentWaitTime) + GinkgoWriter.Printf(" Pipeline Validation: %v\n", f.metrics.PipelineValidationTime) + GinkgoWriter.Printf(" Cleanup: %v\n", f.metrics.CleanupTime) + GinkgoWriter.Printf(" Total: %v\n", f.metrics.SetupTime+f.metrics.DeploymentWaitTime+f.metrics.PipelineValidationTime+f.metrics.CleanupTime) +} + +// splitFields splits space-separated fields +func splitFields(s string) []string { + return strings.Fields(s) +} + +// GetPodLogs retrieves logs from a pod +func (f *Framework) GetPodLogs(podName string) (string, error) { + return f.kubectl.GetPodLogs(podName) +} + +// GetPodLogsTail retrieves the last N lines of logs from a pod +func (f *Framework) GetPodLogsTail(podName string, lines int) (string, error) { + return f.kubectl.GetPodLogsTail(podName, lines) +} + +// GetPodsByLabel retrieves pod names matching a label selector +func (f *Framework) GetPodsByLabel(labelSelector string) ([]string, error) { + return f.kubectl.GetPodsByLabel(labelSelector) +} + +// WaitForPodReady waits for a pod to become ready +func (f *Framework) WaitForPodReady(podName string) { + By(fmt.Sprintf("waiting for pod %s to be ready", podName)) + err := f.kubectl.WaitForPodReady(podName, "2m") + Expect(err).NotTo(HaveOccurred(), "Pod %s did not become ready in namespace %s", podName, f.namespace) +} + +// GetAggregatorPods retrieves aggregator pod names for a given aggregator +func (f *Framework) GetAggregatorPods(aggregatorName string) ([]string, error) { + // Aggregator pods use instance label to identify which aggregator they belong to + labelSelector := fmt.Sprintf("app.kubernetes.io/instance=%s,app.kubernetes.io/component=Aggregator", aggregatorName) + return f.kubectl.GetPodsByLabel(labelSelector) +} + +// GetAgentPods retrieves agent pod names +func (f *Framework) GetAgentPods(vectorName string) ([]string, error) { + // Agent pods use instance label and component=Agent + labelSelector := fmt.Sprintf("app.kubernetes.io/instance=%s,app.kubernetes.io/component=Agent", vectorName) + return f.kubectl.GetPodsByLabel(labelSelector) +} + +// GetPipelineAnnotation retrieves a specific annotation from a pipeline +func (f *Framework) GetPipelineAnnotation(name string, annotationKey string) string { + jsonPath := fmt.Sprintf(".metadata.annotations['%s']", annotationKey) + result, err := f.kubectl.GetWithJsonPath("vectorpipeline", name, jsonPath) + if err != nil { + // Annotation might not exist, which is expected in some cases + return "" + } + return result +} + +// VerifyAgentHasPipeline verifies that the agent Secret contains the specified pipeline +func (f *Framework) VerifyAgentHasPipeline(vectorName, pipelineName string) error { + return f.VerifyAgentHasPipelineInNamespace(vectorName, pipelineName, f.namespace) +} + +// VerifyAgentHasPipelineInNamespace verifies that an agent Secret contains the specified pipeline from a specific namespace +func (f *Framework) VerifyAgentHasPipelineInNamespace(vectorName, pipelineName, namespace string) error { + // Get the agent's vector config from the Secret + // The config is stored in a Secret with name pattern: {vectorName}-agent + secretName := fmt.Sprintf("%s-agent", vectorName) + + // Get base64-encoded config from Secret + encodedConfig, err := f.kubectl.GetWithJsonPath("secret", secretName, ".data['agent\\.json']") + if err != nil { + return fmt.Errorf("failed to get agent secret %s: %w", secretName, err) + } + + if encodedConfig == "" { + return fmt.Errorf("agent secret %s has no agent.json data", secretName) + } + + // Check size before decoding to prevent DoS via large payloads + maxEncodedSize := MaxConfigSize * 4 / 3 + if len(encodedConfig) > maxEncodedSize { + return fmt.Errorf("config too large: %d bytes (max %d bytes)", len(encodedConfig), maxEncodedSize) + } + + // Decode base64 + configBytes, err := base64.StdEncoding.DecodeString(encodedConfig) + if err != nil { + return fmt.Errorf("failed to decode base64 config from secret %s: %w", secretName, err) + } + config := string(configBytes) + + if config == "" { + return fmt.Errorf("agent config is empty after decoding") + } + + // Check if the pipeline name appears in the config + // In normal mode, pipeline components are prefixed with namespace-pipelinename- + expectedPrefix := fmt.Sprintf("%s-%s-", namespace, pipelineName) + if !strings.Contains(config, expectedPrefix) { + return fmt.Errorf("pipeline %s not found in agent config (expected prefix: %s)", pipelineName, expectedPrefix) + } + + return nil +} + +// VerifyAgentHasClusterPipeline verifies that an agent Secret contains the specified ClusterVectorPipeline +func (f *Framework) VerifyAgentHasClusterPipeline(vectorName, pipelineName string) error { + // Get the agent's vector config from the Secret + secretName := fmt.Sprintf("%s-agent", vectorName) + + // Get base64-encoded config from Secret + encodedConfig, err := f.kubectl.GetWithJsonPath("secret", secretName, ".data['agent\\.json']") + if err != nil { + return fmt.Errorf("failed to get agent secret %s: %w", secretName, err) + } + + if encodedConfig == "" { + return fmt.Errorf("agent secret %s has no agent.json data", secretName) + } + + // Check size before decoding to prevent DoS via large payloads + maxEncodedSize := MaxConfigSize * 4 / 3 + if len(encodedConfig) > maxEncodedSize { + return fmt.Errorf("config too large: %d bytes (max %d bytes)", len(encodedConfig), maxEncodedSize) + } + + // Decode base64 + configBytes, err := base64.StdEncoding.DecodeString(encodedConfig) + if err != nil { + return fmt.Errorf("failed to decode base64 config from secret %s: %w", secretName, err) + } + config := string(configBytes) + + if config == "" { + return fmt.Errorf("agent config is empty after decoding") + } + + // Check if the cluster pipeline name appears in the config + // ClusterVectorPipeline components are prefixed with only pipelinename- (no namespace prefix) + expectedPrefix := fmt.Sprintf("%s-", pipelineName) + if !strings.Contains(config, expectedPrefix) { + return fmt.Errorf("cluster pipeline %s not found in agent config (expected prefix: %s)", pipelineName, expectedPrefix) + } + + return nil +} + +// VerifyAggregatorHasPipeline verifies that an aggregator Secret contains the specified pipeline +func (f *Framework) VerifyAggregatorHasPipeline(aggregatorName, pipelineName string) error { + // Get the aggregator's vector config from the Secret + // The config is stored in a Secret with name pattern: {aggregatorName}-aggregator + secretName := fmt.Sprintf("%s-aggregator", aggregatorName) + + // Get base64-encoded config from Secret + encodedConfig, err := f.kubectl.GetWithJsonPath("secret", secretName, ".data['config\\.json']") + if err != nil { + return fmt.Errorf("failed to get aggregator secret %s: %w", secretName, err) + } + + if encodedConfig == "" { + return fmt.Errorf("aggregator secret %s has no config.json data", secretName) + } + + // Check size before decoding to prevent DoS via large payloads + maxEncodedSize := MaxConfigSize * 4 / 3 + if len(encodedConfig) > maxEncodedSize { + return fmt.Errorf("config too large: %d bytes (max %d bytes)", len(encodedConfig), maxEncodedSize) + } + + // Decode base64 + configBytes, err := base64.StdEncoding.DecodeString(encodedConfig) + if err != nil { + return fmt.Errorf("failed to decode base64 config from secret %s: %w", secretName, err) + } + config := string(configBytes) + + if config == "" { + return fmt.Errorf("aggregator %s config is empty after decoding", aggregatorName) + } + + // Check if the pipeline name appears in the config + expectedPrefix := fmt.Sprintf("%s-%s-", f.namespace, pipelineName) + if !strings.Contains(config, expectedPrefix) { + return fmt.Errorf("pipeline %s not found in aggregator %s config (expected prefix: %s)", + pipelineName, aggregatorName, expectedPrefix) + } + + return nil +} + +// ApplyTestDataWithVars loads and applies a test manifest with variable substitution +func (f *Framework) ApplyTestDataWithVars(path string, vars map[string]string) { + By(fmt.Sprintf("applying test data with vars: %s", path)) + + content, err := os.ReadFile(filepath.Join(f.TestDataPath, path)) + Expect(err).NotTo(HaveOccurred(), "Failed to load test data from %s", path) + + // Replace namespace in YAML + yamlContent := replaceNamespace(string(content), f.namespace) + + // Replace variables + for placeholder, value := range vars { + yamlContent = strings.ReplaceAll(yamlContent, placeholder, value) + } + + err = f.kubectl.Apply(yamlContent) + Expect(err).NotTo(HaveOccurred(), "Failed to apply test data %s in namespace %s", path, f.namespace) +} + +// DeleteResource deletes a Kubernetes resource +func (f *Framework) DeleteResource(kind, name string) { + By(fmt.Sprintf("deleting %s %s", kind, name)) + err := f.kubectl.Delete(kind, name) + Expect(err).NotTo(HaveOccurred(), "Failed to delete %s %s in namespace %s", kind, name, f.namespace) +} + +// WaitForPodReadyInNamespace waits for a pod to become ready in a specific namespace +func (f *Framework) WaitForPodReadyInNamespace(podName, namespace string) { + By(fmt.Sprintf("waiting for pod %s to be ready in namespace %s", podName, namespace)) + client := kubectl.NewClient(namespace) + err := client.WaitForPodReady(podName, "2m") + Expect(err).NotTo(HaveOccurred(), "Pod %s did not become ready in namespace %s", podName, namespace) +} + +// WaitForPipelineValidInNamespace waits for a pipeline to become valid in a specific namespace +func (f *Framework) WaitForPipelineValidInNamespace(name, namespace string) { + By(fmt.Sprintf("waiting for pipeline %s to become valid in namespace %s", name, namespace)) + start := time.Now() + defer func() { + duration := time.Since(start) + GinkgoWriter.Printf("⏱️ Pipeline %s validated in %v (namespace: %s)\n", name, duration, namespace) + }() + + client := kubectl.NewClient(namespace) + client.WaitForPipelineValid(name) +} + +// GetPipelineAnnotationInNamespace retrieves a specific annotation from a pipeline in a specific namespace +func (f *Framework) GetPipelineAnnotationInNamespace(name, namespace, annotationKey string) string { + jsonPath := fmt.Sprintf(".metadata.annotations['%s']", annotationKey) + client := kubectl.NewClient(namespace) + result, err := client.GetWithJsonPath("vectorpipeline", name, jsonPath) + if err != nil { + // Annotation might not exist, which is expected in some cases + return "" + } + return result +} + +// WaitForClusterPipelineValid waits for a ClusterVectorPipeline to become valid +func (f *Framework) WaitForClusterPipelineValid(name string) { + By(fmt.Sprintf("waiting for ClusterVectorPipeline %s to become valid", name)) + start := time.Now() + defer func() { + duration := time.Since(start) + GinkgoWriter.Printf("⏱️ ClusterVectorPipeline %s validated in %v\n", name, duration) + }() + + // ClusterVectorPipeline is cluster-scoped, so we use a client without namespace + client := kubectl.NewClient("") + Eventually(func() string { + result, _ := client.GetWithJsonPath("clustervectorpipeline", name, ".status.configCheckResult") + return result + }, config.PipelineValidTimeout, config.DefaultPollInterval).Should(Equal("true"), + "ClusterVectorPipeline %s did not become valid", name) +} + +// GetClusterPipelineAnnotation retrieves a specific annotation from a ClusterVectorPipeline +func (f *Framework) GetClusterPipelineAnnotation(name, annotationKey string) string { + jsonPath := fmt.Sprintf(".metadata.annotations['%s']", annotationKey) + client := kubectl.NewClient("") + result, err := client.GetWithJsonPath("clustervectorpipeline", name, jsonPath) + if err != nil { + // Annotation might not exist, which is expected in some cases + return "" + } + return result +} + +// GetClusterPipelineStatus retrieves a specific status field from a ClusterVectorPipeline +func (f *Framework) GetClusterPipelineStatus(name, field string) string { + client := kubectl.NewClient("") + result, err := client.GetWithJsonPath("clustervectorpipeline", name, fmt.Sprintf(".status.%s", field)) + Expect(err).NotTo(HaveOccurred(), + "Failed to get ClusterVectorPipeline %s status field %s", name, field) + return result +} + +// Kubectl returns the kubectl client +func (f *Framework) Kubectl() *kubectl.Client { + return f.kubectl +} + +// GetRegisteredFramework retrieves a framework by namespace +// Used by artifact collector to access kubectl client and namespace +func GetRegisteredFramework(namespace string) (*Framework, bool) { + value, ok := frameworkRegistry.Load(namespace) + if !ok { + return nil, false + } + return value.(*Framework), true +} + +// GetFrameworkRegistry returns the framework registry for iteration +// Used by ReportAfterEach to find frameworks when namespace is not known +func GetFrameworkRegistry() *sync.Map { + return &frameworkRegistry +} + +// GetSecret retrieves a Secret by name in the framework's namespace +func (f *Framework) GetSecret(name string) (map[string][]byte, error) { + cmd := fmt.Sprintf("kubectl get secret %s -n %s -o json", name, f.namespace) + output, err := exec.Command("sh", "-c", cmd).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to get secret %s: %w, output: %s", name, err, string(output)) + } + + var secret struct { + Data map[string]string `json:"data"` + } + if err := json.Unmarshal(output, &secret); err != nil { + return nil, fmt.Errorf("failed to unmarshal secret: %w", err) + } + + // Decode base64 data + decodedData := make(map[string][]byte) + maxEncodedSize := MaxConfigSize * 4 / 3 + for k, v := range secret.Data { + // Check size before decoding to prevent DoS via large payloads + if len(v) > maxEncodedSize { + return nil, fmt.Errorf("secret data for key %s too large: %d bytes (max %d bytes)", k, len(v), maxEncodedSize) + } + + decoded, err := base64.StdEncoding.DecodeString(v) + if err != nil { + return nil, fmt.Errorf("failed to decode secret data for key %s: %w", k, err) + } + decodedData[k] = decoded + } + + return decodedData, nil +} + +// GetDeployment retrieves a Deployment by name in the framework's namespace +func (f *Framework) GetDeployment(name string) (*DeploymentInfo, error) { + cmd := fmt.Sprintf("kubectl get deployment %s -n %s -o json", name, f.namespace) + output, err := exec.Command("sh", "-c", cmd).CombinedOutput() + if err != nil { + return nil, fmt.Errorf("failed to get deployment %s: %w, output: %s", name, err, string(output)) + } + + var deployment struct { + Spec struct { + Template struct { + Spec struct { + InitContainers []struct { + Name string `json:"name"` + } `json:"initContainers"` + Containers []struct { + Name string `json:"name"` + } `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` + } + if err := json.Unmarshal(output, &deployment); err != nil { + return nil, fmt.Errorf("failed to unmarshal deployment: %w", err) + } + + info := &DeploymentInfo{ + InitContainers: make([]string, 0), + Containers: make([]string, 0), + } + + for _, c := range deployment.Spec.Template.Spec.InitContainers { + info.InitContainers = append(info.InitContainers, c.Name) + } + for _, c := range deployment.Spec.Template.Spec.Containers { + info.Containers = append(info.Containers, c.Name) + } + + return info, nil +} + +// DeploymentInfo contains simplified deployment information +type DeploymentInfo struct { + InitContainers []string + Containers []string +} + +// RecordStep records a test step for reproducibility +func (f *Framework) RecordStep(step recorder.TestStep) { + if f.recorder != nil { + f.recorder.RecordStep(step) + } +} + +// SetTestName sets the current test name in the recorder +func (f *Framework) SetTestName(name string) { + if f.recorder != nil { + f.recorder.SetTestName(name) + } +} + +// ExportTestPlan exports the recorded test plan to files +func (f *Framework) ExportTestPlan() { + if f.recorder == nil { + return + } + + // Get current test spec info + spec := CurrentSpecReport() + testName := buildTestName(spec) + + if testName == "" { + testName = "unknown-test" + } + + f.recorder.SetTestName(testName) + + // In dry-run mode, print to stdout + if f.dryRun { + fmt.Println("\n" + strings.Repeat("=", 80)) + fmt.Printf("Test Plan: %s\n", testName) + fmt.Println(strings.Repeat("=", 80)) + fmt.Println(f.recorder.ExportAsShellScript()) + return + } + + // Otherwise, save to artifact directory if it exists + artifactDir := os.Getenv("ARTIFACT_DIR") + if artifactDir == "" { + artifactDir = "test/e2e/results/test-plans" + } + + // Create directory if it doesn't exist + if err := os.MkdirAll(artifactDir, 0755); err != nil { + fmt.Printf("Warning: failed to create artifact directory: %v\n", err) + return + } + + // Sanitize test name for filename + safeTestName := strings.ReplaceAll(testName, " ", "-") + safeTestName = strings.ReplaceAll(safeTestName, "/", "-") + + // Save as shell script + scriptPath := filepath.Join(artifactDir, fmt.Sprintf("%s.sh", safeTestName)) + scriptContent := f.recorder.ExportAsShellScript() + if err := os.WriteFile(scriptPath, []byte(scriptContent), 0755); err != nil { + fmt.Printf("Warning: failed to write test plan script: %v\n", err) + } else { + fmt.Printf("✓ Test plan saved to: %s\n", scriptPath) + } + + // Save as markdown + mdPath := filepath.Join(artifactDir, fmt.Sprintf("%s.md", safeTestName)) + mdContent := f.recorder.ExportAsMarkdown() + if err := os.WriteFile(mdPath, []byte(mdContent), 0644); err != nil { + fmt.Printf("Warning: failed to write test plan markdown: %v\n", err) + } else { + fmt.Printf("✓ Test plan documentation saved to: %s\n", mdPath) + } +} + +// buildTestName constructs a test name from the spec report +func buildTestName(spec types.SpecReport) string { + hierarchy := spec.ContainerHierarchyTexts + leaf := spec.LeafNodeText + + if len(hierarchy) > 0 { + return strings.Join(append(hierarchy, leaf), " ") + } + return leaf +} + +// ToContext stores the framework in the given context +// This allows framework to be passed through context chains if needed +func (f *Framework) ToContext(ctx context.Context) context.Context { + return context.WithValue(ctx, FrameworkContextKey{}, f) +} + +// FromContext retrieves a framework from the given context +// Returns nil if no framework is stored in the context +func FromContext(ctx context.Context) *Framework { + if f, ok := ctx.Value(FrameworkContextKey{}).(*Framework); ok { + return f + } + return nil +} + +// FromReportEntries retrieves a framework from Ginkgo report entries +// This is the preferred way to access framework in ReportAfterEach +// Returns nil if no framework entry is found +func FromReportEntries(entries []types.ReportEntry) *Framework { + for _, entry := range entries { + if entry.Name == frameworkReportEntryName { + // GetRawValue() returns the underlying interface{} value + if f, ok := entry.Value.GetRawValue().(*Framework); ok { + return f + } + } + } + return nil +} + +// LogOptions contains options for retrieving pod logs +type LogOptions struct { + // Container name to get logs from (empty for default container) + Container string + // TailLines limits the number of lines from the end of the logs + TailLines int + // SinceSeconds returns logs newer than a relative duration (in seconds) + SinceSeconds int +} + +// WaitForLogsContaining waits for a substring to appear in pod logs +// Returns nil if found, error if timeout occurs +func (f *Framework) WaitForLogsContaining(podName, substring string, timeout time.Duration) error { + fmt.Fprintf(GinkgoWriter, "⏳ Waiting for logs in pod %s to contain: %s\n", podName, substring) + + var lastLogs string + startTime := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + err := wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + logs, err := f.GetPodLogs(podName) + if err != nil { + // Not a critical error, pod might not exist yet or be starting + return false, nil + } + lastLogs = logs + return strings.Contains(logs, substring), nil + }) + + if err != nil { + elapsed := time.Since(startTime) + // Truncate logs if too long + truncatedLogs := lastLogs + if len(lastLogs) > 500 { + truncatedLogs = lastLogs[len(lastLogs)-500:] + "\n... (truncated)" + } + return fmt.Errorf("timeout waiting for logs to contain '%s' in pod %s after %v. Last logs:\n%s", + substring, podName, elapsed, truncatedLogs) + } + + elapsed := time.Since(startTime) + fmt.Fprintf(GinkgoWriter, "✓ Found expected substring in pod %s logs (took %v)\n", podName, elapsed) + return nil +} + +// WaitForLogsMatching waits for a regex pattern to match in pod logs +// Returns nil if match found, error if timeout occurs or pattern is invalid +func (f *Framework) WaitForLogsMatching(podName, pattern string, timeout time.Duration) error { + fmt.Fprintf(GinkgoWriter, "⏳ Waiting for logs in pod %s to match pattern: %s\n", podName, pattern) + + // Compile regex pattern + re, err := regexp.Compile(pattern) + if err != nil { + return fmt.Errorf("invalid regex pattern '%s': %w", pattern, err) + } + + var lastLogs string + startTime := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + err = wait.PollUntilContextTimeout(ctx, time.Second, timeout, true, func(ctx context.Context) (bool, error) { + logs, err := f.GetPodLogs(podName) + if err != nil { + // Not a critical error, pod might not exist yet or be starting + return false, nil + } + lastLogs = logs + return re.MatchString(logs), nil + }) + + if err != nil { + elapsed := time.Since(startTime) + // Truncate logs if too long + truncatedLogs := lastLogs + if len(lastLogs) > 500 { + truncatedLogs = lastLogs[len(lastLogs)-500:] + "\n... (truncated)" + } + return fmt.Errorf("timeout waiting for logs to match pattern '%s' in pod %s after %v. Last logs:\n%s", + pattern, podName, elapsed, truncatedLogs) + } + + elapsed := time.Since(startTime) + fmt.Fprintf(GinkgoWriter, "✓ Found pattern match in pod %s logs (took %v)\n", podName, elapsed) + return nil +} + +// AssertNoLogsContaining verifies that a substring does NOT appear in pod logs +// Returns nil if substring is absent for the entire check duration, error otherwise +func (f *Framework) AssertNoLogsContaining(podName, substring string, checkDuration time.Duration) error { + fmt.Fprintf(GinkgoWriter, "⏳ Verifying logs in pod %s do NOT contain: %s (checking for %v)\n", + podName, substring, checkDuration) + + var foundLogs string + startTime := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), checkDuration) + defer cancel() + + err := wait.PollUntilContextTimeout(ctx, time.Second, checkDuration, true, func(ctx context.Context) (bool, error) { + logs, err := f.GetPodLogs(podName) + if err != nil { + // Pod might not exist yet, which is acceptable for negative checks + return false, nil + } + + if strings.Contains(logs, substring) { + foundLogs = logs + // Found the substring - this is a failure for negative assertion + return true, nil + } + + // Continue checking + return false, nil + }) + + // For Consistently-style checks, we want to ensure the substring was NEVER found + if wait.Interrupted(err) { + // Timeout means we successfully verified absence for the entire duration + elapsed := time.Since(startTime) + fmt.Fprintf(GinkgoWriter, "✓ Verified substring absent in pod %s logs for %v\n", podName, elapsed) + return nil + } + + if foundLogs != "" { + // We found the substring - this is an error + truncatedLogs := foundLogs + if len(foundLogs) > 500 { + truncatedLogs = foundLogs[len(foundLogs)-500:] + "\n... (truncated)" + } + return fmt.Errorf("found unexpected substring '%s' in pod %s logs. Last logs:\n%s", + substring, podName, truncatedLogs) + } + + // Other error occurred + if err != nil { + return fmt.Errorf("error while checking logs for pod %s: %w", podName, err) + } + + return nil +} + +// GetPodLogsWithOptions retrieves logs from a pod with custom options +func (f *Framework) GetPodLogsWithOptions(podName string, opts LogOptions) (string, error) { + if opts.Container != "" || opts.TailLines > 0 || opts.SinceSeconds > 0 { + // Use kubectl client methods if options are specified + if opts.TailLines > 0 { + return f.kubectl.GetPodLogsTail(podName, opts.TailLines) + } + // For other options, we'd need to add more kubectl methods + // For now, fall back to basic GetPodLogs + return f.kubectl.GetPodLogs(podName) + } + + return f.kubectl.GetPodLogs(podName) +} diff --git a/test/e2e/framework/kubectl/client.go b/test/e2e/framework/kubectl/client.go new file mode 100644 index 00000000..6d13ca7f --- /dev/null +++ b/test/e2e/framework/kubectl/client.go @@ -0,0 +1,485 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubectl + +import ( + "encoding/json" + "fmt" + "log" + "os/exec" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + + "github.com/kaasops/vector-operator/test/utils" +) + +// Client provides convenient kubectl operations +type Client struct { + namespace string +} + +// NewClient creates a new kubectl client for the given namespace +func NewClient(namespace string) *Client { + return &Client{namespace: namespace} +} + +// Apply applies YAML content to the cluster with explicit namespace override +// This ensures resources are created in the correct test namespace +func (c *Client) Apply(yamlContent string) error { + // Validate namespace to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return fmt.Errorf("namespace validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl apply -f - -n %s", c.namespace) + + cmd := exec.Command("kubectl", "apply", "-f", "-", "-n", c.namespace) + cmd.Stdin = strings.NewReader(yamlContent) + output, err := utils.Run(cmd) + + // Log kubectl output for debugging (helps catch namespace mismatches) + if len(output) > 0 { + fmt.Printf("kubectl apply: %s\n", string(output)) + } + + return err +} + +// ApplyWithoutNamespaceOverride applies YAML content without forcing namespace +// Use this when the YAML already contains the correct namespace field +func (c *Client) ApplyWithoutNamespaceOverride(yamlContent string) error { + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl apply -f -") + + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(yamlContent) + output, err := utils.Run(cmd) + + // Log kubectl output for debugging + if len(output) > 0 { + fmt.Printf("kubectl apply: %s\n", string(output)) + } + + return err +} + +// Get retrieves a resource by name and type +func (c *Client) Get(resourceType, name string) ([]byte, error) { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return nil, fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateResourceType(resourceType); err != nil { + return nil, fmt.Errorf("resource type validation failed: %w", err) + } + if err := ValidateResourceName(name); err != nil { + return nil, fmt.Errorf("resource name validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl get %s %s -n %s", resourceType, name, c.namespace) + + cmd := exec.Command("kubectl", "get", resourceType, name, "-n", c.namespace) + return utils.Run(cmd) +} + +// GetWithJsonPath retrieves a specific field from a resource +// If name is empty, retrieves from all resources of the given type +func (c *Client) GetWithJsonPath(resourceType, name, jsonPath string) (string, error) { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return "", fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateResourceType(resourceType); err != nil { + return "", fmt.Errorf("resource type validation failed: %w", err) + } + if name != "" { + if err := ValidateResourceName(name); err != nil { + return "", fmt.Errorf("resource name validation failed: %w", err) + } + } + if err := ValidateJSONPath(jsonPath); err != nil { + return "", fmt.Errorf("jsonPath validation failed: %w", err) + } + + // Build command args based on whether name is specified + args := []string{"get", resourceType} + + // Only include name if it's not empty (empty name means get all resources) + if name != "" { + args = append(args, name) + } + + args = append(args, "-n", c.namespace, "-o", fmt.Sprintf("jsonpath={%s}", jsonPath)) + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl %s", strings.Join(args, " ")) + + cmd := exec.Command("kubectl", args...) + output, err := utils.Run(cmd) + return string(output), err +} + +// GetAll retrieves all resources of a type with optional label selector +func (c *Client) GetAll(resourceType string, labelSelector string) (string, error) { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return "", fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateResourceType(resourceType); err != nil { + return "", fmt.Errorf("resource type validation failed: %w", err) + } + if labelSelector != "" { + if err := ValidateLabelSelector(labelSelector); err != nil { + return "", fmt.Errorf("label selector validation failed: %w", err) + } + } + + args := []string{"get", resourceType, "-n", c.namespace} + if labelSelector != "" { + args = append(args, "-l", labelSelector) + } + args = append(args, "-o", "jsonpath={.items[*].metadata.name}") + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl %s", strings.Join(args, " ")) + + cmd := exec.Command("kubectl", args...) + output, err := utils.Run(cmd) + return string(output), err +} + +// Wait waits for a resource condition +func (c *Client) Wait(resourceType, name, condition string, timeout string) error { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateResourceType(resourceType); err != nil { + return fmt.Errorf("resource type validation failed: %w", err) + } + if err := ValidateResourceName(name); err != nil { + return fmt.Errorf("resource name validation failed: %w", err) + } + if err := ValidateTimeout(timeout); err != nil { + return fmt.Errorf("timeout validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl wait --for=%s --timeout=%s %s/%s -n %s", condition, timeout, resourceType, name, c.namespace) + + cmd := exec.Command("kubectl", "wait", + fmt.Sprintf("--for=%s", condition), + fmt.Sprintf("--timeout=%s", timeout), + fmt.Sprintf("%s/%s", resourceType, name), + "-n", c.namespace) + _, err := utils.Run(cmd) + return err +} + +// Delete deletes a resource +func (c *Client) Delete(resourceType, name string) error { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateResourceType(resourceType); err != nil { + return fmt.Errorf("resource type validation failed: %w", err) + } + if err := ValidateResourceName(name); err != nil { + return fmt.Errorf("resource name validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl delete %s %s -n %s", resourceType, name, c.namespace) + + cmd := exec.Command("kubectl", "delete", resourceType, name, "-n", c.namespace) + _, err := utils.Run(cmd) + return err +} + +// CreateNamespace creates a namespace +func CreateNamespace(name string) error { + // Validate namespace to prevent command injection + if err := ValidateNamespace(name); err != nil { + return fmt.Errorf("namespace validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl create ns %s", name) + + cmd := exec.Command("kubectl", "create", "ns", name) + _, err := utils.Run(cmd) + return err +} + +// GetNamespace retrieves namespace information +func GetNamespace(name string) (*corev1.Namespace, error) { + // Validate namespace to prevent command injection + if err := ValidateNamespace(name); err != nil { + return nil, fmt.Errorf("namespace validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl get ns %s -o json", name) + + cmd := exec.Command("kubectl", "get", "ns", name, "-o", "json") + output, err := cmd.Output() + if err != nil { + return nil, err + } + + var ns corev1.Namespace + if err := json.Unmarshal(output, &ns); err != nil { + return nil, fmt.Errorf("failed to parse namespace JSON: %w", err) + } + + return &ns, nil +} + +// DeleteNamespace deletes a namespace with retry and force delete fallback +// Handles CRD resources with finalizers to prevent stuck namespaces +func DeleteNamespace(name string, timeout string) error { + // Validate parameters to prevent command injection + if err := ValidateNamespace(name); err != nil { + return fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateTimeout(timeout); err != nil { + return fmt.Errorf("timeout validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl delete ns %s --timeout=%s", name, timeout) + + // Parse timeout duration for wait logic + timeoutDuration, err := parseDuration(timeout) + if err != nil { + return fmt.Errorf("invalid timeout format: %w", err) + } + + // First try: normal delete + cmd := exec.Command("kubectl", "delete", "ns", name, fmt.Sprintf("--timeout=%s", timeout)) + _, err = utils.Run(cmd) + if err == nil { + // Wait for namespace to actually disappear + return waitForNamespaceDeletion(name, timeoutDuration) + } + + // If normal delete fails or times out, force cleanup CRD resources first + fmt.Printf("⚠️ Namespace %s deletion failed, attempting force cleanup\n", name) + + // Clean up operator CRD resources that might have finalizers + crdTypes := []string{ + "vectorpipeline", + "vectoraggregator", + "vector", + "clustervectorpipeline", + "clustervectoraggregator", + } + + for _, crdType := range crdTypes { + // Get all resources of this type + cmd := exec.Command("kubectl", "get", crdType, "-n", name, "-o", "name") + output, err := cmd.Output() + if err != nil { + continue // Resource type doesn't exist or no resources, skip + } + + resources := strings.Fields(string(output)) + for _, resource := range resources { + // Remove finalizers + patchCmd := exec.Command("kubectl", "patch", resource, "-n", name, + "-p", `{"metadata":{"finalizers":[]}}`, + "--type=merge") + _ = patchCmd.Run() // Ignore errors + + // Force delete + deleteCmd := exec.Command("kubectl", "delete", resource, "-n", name, + "--grace-period=0", "--force") + _ = deleteCmd.Run() // Ignore errors + } + } + + // Remove namespace finalizers + _ = exec.Command("kubectl", "patch", "ns", name, + "-p", `{"metadata":{"finalizers":[]}}`, + "--type=merge").Run() + + // Then force delete namespace with shorter timeout + log.Printf("KUBECTL_CMD: kubectl delete ns %s --grace-period=0 --force --timeout=10s", name) + cmd = exec.Command("kubectl", "delete", "ns", name, + "--grace-period=0", "--force", "--timeout=10s") + _, _ = utils.Run(cmd) + + // Wait for namespace to actually disappear, even after force delete + waitErr := waitForNamespaceDeletion(name, 30*time.Second) + if waitErr != nil { + fmt.Printf("⚠️ Namespace %s still exists after cleanup, continuing anyway\n", name) + return nil // Don't fail the test - namespace will be cleaned up eventually + } + + return nil +} + +// waitForNamespaceDeletion waits for a namespace to be fully deleted +func waitForNamespaceDeletion(name string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + pollInterval := 2 * time.Second + + for time.Now().Before(deadline) { + // Try to get the namespace + cmd := exec.Command("kubectl", "get", "ns", name) + err := cmd.Run() + if err != nil { + // Namespace not found - deletion successful + log.Printf("KUBECTL_CMD: namespace %s successfully deleted", name) + return nil + } + + // Namespace still exists, wait and retry + time.Sleep(pollInterval) + } + + return fmt.Errorf("namespace %s still exists after %v", name, timeout) +} + +// parseDuration parses timeout strings like "30s", "5m", "1h" +func parseDuration(timeout string) (time.Duration, error) { + // Extract numeric part and unit + if len(timeout) < 2 { + return 0, fmt.Errorf("invalid timeout: %s", timeout) + } + + unit := timeout[len(timeout)-1:] + valueStr := timeout[:len(timeout)-1] + + var value int + _, err := fmt.Sscanf(valueStr, "%d", &value) + if err != nil { + return 0, fmt.Errorf("invalid timeout value: %s", timeout) + } + + switch unit { + case "s": + return time.Duration(value) * time.Second, nil + case "m": + return time.Duration(value) * time.Minute, nil + case "h": + return time.Duration(value) * time.Hour, nil + default: + return 0, fmt.Errorf("invalid timeout unit: %s (must be s, m, or h)", unit) + } +} + +// GetPodLogs retrieves logs from a pod +func (c *Client) GetPodLogs(podName string) (string, error) { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return "", fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateResourceName(podName); err != nil { + return "", fmt.Errorf("pod name validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl logs %s -n %s", podName, c.namespace) + + cmd := exec.Command("kubectl", "logs", podName, "-n", c.namespace) + output, err := utils.Run(cmd) + return string(output), err +} + +// GetPodLogsSince retrieves logs from a pod since a specific time +func (c *Client) GetPodLogsSince(podName string, since string) (string, error) { + cmd := exec.Command("kubectl", "logs", podName, "-n", c.namespace, "--since", since) + output, err := utils.Run(cmd) + return string(output), err +} + +// GetPodLogsTail retrieves the last N lines of logs from a pod +func (c *Client) GetPodLogsTail(podName string, lines int) (string, error) { + cmd := exec.Command("kubectl", "logs", podName, "-n", c.namespace, "--tail", fmt.Sprintf("%d", lines)) + output, err := utils.Run(cmd) + return string(output), err +} + +// GetPodLogsSinceTime retrieves logs from a pod since a specific time with line limit +// Uses --since-time for temporal filtering and --tail as a safety limit +func (c *Client) GetPodLogsSinceTime(podName string, since time.Time, tailLines int) (string, error) { + // Format time as RFC3339 for Kubernetes + sinceTime := since.Format(time.RFC3339) + + // Use both --since-time and --tail: + // --since-time filters logs by timestamp + // --tail provides safety limit if too many logs match + cmd := exec.Command("kubectl", "logs", podName, "-n", c.namespace, + "--since-time", sinceTime, + "--tail", fmt.Sprintf("%d", tailLines)) + output, err := utils.Run(cmd) + return string(output), err +} + +// GetPodsByLabel retrieves pod names matching a label selector +func (c *Client) GetPodsByLabel(labelSelector string) ([]string, error) { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return nil, fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateLabelSelector(labelSelector); err != nil { + return nil, fmt.Errorf("label selector validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl get pods -n %s -l %s -o jsonpath={.items[*].metadata.name}", c.namespace, labelSelector) + + cmd := exec.Command("kubectl", "get", "pods", "-n", c.namespace, "-l", labelSelector, "-o", "jsonpath={.items[*].metadata.name}") + output, err := utils.Run(cmd) + if err != nil { + return nil, err + } + + podNames := strings.Fields(string(output)) + return podNames, nil +} + +// WaitForPodReady waits for a pod to become ready +func (c *Client) WaitForPodReady(podName string, timeout string) error { + // Validate parameters to prevent command injection + if err := ValidateNamespace(c.namespace); err != nil { + return fmt.Errorf("namespace validation failed: %w", err) + } + if err := ValidateResourceName(podName); err != nil { + return fmt.Errorf("pod name validation failed: %w", err) + } + if err := ValidateTimeout(timeout); err != nil { + return fmt.Errorf("timeout validation failed: %w", err) + } + + // Log command for audit and reproducibility + log.Printf("KUBECTL_CMD: kubectl wait --for=condition=Ready --timeout=%s pod/%s -n %s", timeout, podName, c.namespace) + + cmd := exec.Command("kubectl", "wait", + "--for=condition=Ready", + fmt.Sprintf("--timeout=%s", timeout), + fmt.Sprintf("pod/%s", podName), + "-n", c.namespace) + _, err := utils.Run(cmd) + return err +} diff --git a/test/e2e/framework/kubectl/validation.go b/test/e2e/framework/kubectl/validation.go new file mode 100644 index 00000000..40986df7 --- /dev/null +++ b/test/e2e/framework/kubectl/validation.go @@ -0,0 +1,143 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubectl + +import ( + "fmt" + "regexp" +) + +// ValidateNamespace validates namespace against RFC 1123 DNS Label requirements. +// A valid namespace must: +// - Be 1-63 characters long +// - Contain only lowercase alphanumeric characters or '-' +// - Start with an alphanumeric character +// - End with an alphanumeric character +// Empty namespace is allowed for cluster-scoped resources +func ValidateNamespace(namespace string) error { + // Allow empty namespace for cluster-scoped resources + if len(namespace) == 0 { + return nil + } + + if len(namespace) > 63 { + return fmt.Errorf("namespace length must be 1-63 characters, got %d", len(namespace)) + } + + // RFC 1123 DNS Label regex: lowercase alphanumeric and hyphens only + // Must start and end with alphanumeric + if !regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`).MatchString(namespace) { + return fmt.Errorf("invalid namespace format: %s (must match RFC 1123 DNS Label)", namespace) + } + + return nil +} + +// ValidateResourceName validates Kubernetes resource names against RFC 1123 DNS Subdomain requirements. +// A valid resource name must: +// - Be 1-253 characters long +// - Contain only lowercase alphanumeric characters, '-', or '.' +// - Start with an alphanumeric character +// - End with an alphanumeric character +func ValidateResourceName(name string) error { + if len(name) == 0 { + return fmt.Errorf("resource name cannot be empty") + } + + if len(name) > 253 { + return fmt.Errorf("resource name length must be 1-253 characters, got %d", len(name)) + } + + // RFC 1123 DNS Subdomain regex + if !regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`).MatchString(name) { + return fmt.Errorf("invalid resource name format: %s (must match RFC 1123 DNS Subdomain)", name) + } + + return nil +} + +// ValidateResourceType validates Kubernetes resource type names. +// These are typically lowercase and may contain '.' for API groups. +func ValidateResourceType(resourceType string) error { + if len(resourceType) == 0 { + return fmt.Errorf("resource type cannot be empty") + } + + // Allow alphanumeric, dots for API groups (e.g., "apps.deployment") + if !regexp.MustCompile(`^[a-z0-9]([a-z0-9\.\-]*[a-z0-9])?$`).MatchString(resourceType) { + return fmt.Errorf("invalid resource type format: %s", resourceType) + } + + return nil +} + +// ValidateLabelSelector validates Kubernetes label selectors. +// Label selectors have specific syntax requirements for keys and values. +func ValidateLabelSelector(selector string) error { + if selector == "" { + // Empty selector is valid (means no filter) + return nil + } + + // Basic validation: check for suspicious characters that could be used for injection + // Allow alphanumeric, dots, hyphens, underscores, slashes (for label keys), equals, commas + if !regexp.MustCompile(`^[a-zA-Z0-9\.\_\-/=,]+$`).MatchString(selector) { + return fmt.Errorf("invalid label selector format: %s", selector) + } + + return nil +} + +// ValidateTimeout validates timeout strings used with kubectl commands. +// Valid formats: "30s", "5m", "1h", "2m0s", "1h30m", "1h30m45s" +// Accepts both simple format (5m) and Go duration format (5m0s) +func ValidateTimeout(timeout string) error { + if timeout == "" { + return fmt.Errorf("timeout cannot be empty") + } + + // Allow Go duration format: combinations of hours, minutes, seconds + // Examples: 30s, 5m, 1h, 2m0s, 1h30m, 1h30m45s + // Pattern: optional hours (Nh), optional minutes (Nm), optional seconds (Ns) + if !regexp.MustCompile(`^([0-9]+h)?([0-9]+m)?([0-9]+(\.[0-9]+)?[sµμn]s?)?$`).MatchString(timeout) { + return fmt.Errorf("invalid timeout format: %s (must be Go duration like '30s', '5m', '2m0s', or '1h30m')", timeout) + } + + // Ensure at least one component is present + if !regexp.MustCompile(`[0-9]`).MatchString(timeout) { + return fmt.Errorf("invalid timeout format: %s (must contain at least one time component)", timeout) + } + + return nil +} + +// ValidateJSONPath validates JSONPath expressions used with kubectl. +// This is a basic validation to prevent obvious injection attempts. +func ValidateJSONPath(jsonPath string) error { + if jsonPath == "" { + return fmt.Errorf("jsonPath cannot be empty") + } + + // Basic validation: JSONPath should not contain shell metacharacters + // Allow alphanumeric, dots, brackets, quotes, underscores, hyphens, asterisks, colons, backslashes + // Backslash is needed for escaping dots in keys like .data['agent\.json'] + if !regexp.MustCompile(`^[\w\.\[\]\{\}'":\*\-\s,@\?\\]+$`).MatchString(jsonPath) { + return fmt.Errorf("invalid jsonPath format: %s", jsonPath) + } + + return nil +} diff --git a/test/e2e/framework/kubectl/wait.go b/test/e2e/framework/kubectl/wait.go new file mode 100644 index 00000000..cfaed942 --- /dev/null +++ b/test/e2e/framework/kubectl/wait.go @@ -0,0 +1,111 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubectl + +import ( + "fmt" + "os/exec" + "strings" + "time" + + . "github.com/onsi/gomega" + + "github.com/kaasops/vector-operator/test/e2e/framework/config" + "github.com/kaasops/vector-operator/test/utils" +) + +// WaitForDeploymentReady waits for a deployment to be created and ready +func (c *Client) WaitForDeploymentReady(name string) { + // First wait for deployment to exist (with reduced timeout) + Eventually(func() error { + cmd := exec.Command("kubectl", "get", "deployment", name, "-n", c.namespace) + _, err := utils.Run(cmd) + return err + }, config.DeploymentCreateTimeout, config.DefaultPollInterval).Should(Succeed(), + "Deployment %s should be created in namespace %s", name, c.namespace) + + // Then wait for it to be available + err := c.Wait("deployment", name, "condition=available", config.DeploymentReadyTimeout.String()) + Expect(err).NotTo(HaveOccurred(), + "Deployment %s should become ready in namespace %s", name, c.namespace) +} + +// WaitForPipelineValid waits for a VectorPipeline to become valid +func (c *Client) WaitForPipelineValid(name string) { + Eventually(func() error { + result, err := c.GetWithJsonPath("vectorpipeline", name, ".status.configCheckResult") + if err != nil { + return err + } + if result != "true" { + return fmt.Errorf("pipeline not valid yet: %s", result) + } + return nil + }, config.PipelineValidTimeout, config.SlowPollInterval).Should(Succeed(), + "Pipeline %s should become valid in namespace %s", name, c.namespace) +} + +// WaitForPipelineInvalid waits for a VectorPipeline to become invalid (for negative tests) +func (c *Client) WaitForPipelineInvalid(name string) { + Eventually(func() error { + result, err := c.GetWithJsonPath("vectorpipeline", name, ".status.configCheckResult") + if err != nil { + return err + } + if result != "false" { + return fmt.Errorf("expected pipeline to be invalid, got: %s", result) + } + return nil + }, config.PipelineValidTimeout, config.SlowPollInterval).Should(Succeed(), + "Pipeline %s should become invalid in namespace %s", name, c.namespace) +} + +// WaitForServiceExists waits for a service to be created +func (c *Client) WaitForServiceExists(name string) { + Eventually(func() error { + _, err := c.Get("service", name) + return err + }, config.ServiceCreateTimeout, config.SlowPollInterval).Should(Succeed(), + "Service %s should be created in namespace %s", name, c.namespace) +} + +// WaitForServiceCount waits for a specific number of services matching filter +func (c *Client) WaitForServiceCount(labelSelector string, expectedCount int, timeout time.Duration) { + Eventually(func() (int, error) { + result, err := c.GetAll("service", labelSelector) + if err != nil { + return 0, err + } + if result == "" { + return 0, nil + } + + services := 0 + for _, svc := range splitFields(result) { + if svc != "" { + services++ + } + } + return services, nil + }, timeout, config.SlowPollInterval).Should(Equal(expectedCount), + "Expected %d services with label %s in namespace %s", expectedCount, labelSelector, c.namespace) +} + +// splitFields splits space-separated fields and filters empty strings +func splitFields(s string) []string { + return strings.Fields(s) +} diff --git a/test/e2e/framework/lifecycle.go b/test/e2e/framework/lifecycle.go new file mode 100644 index 00000000..d7af9475 --- /dev/null +++ b/test/e2e/framework/lifecycle.go @@ -0,0 +1,91 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "time" + + . "github.com/onsi/ginkgo/v2" + + "github.com/kaasops/vector-operator/test/utils" +) + +// SharedDependencies manages dependencies that are shared across all tests +type SharedDependencies struct { + prometheusInstalled bool + certManagerInstalled bool + installTime time.Duration +} + +var globalDeps *SharedDependencies + +// InstallSharedDependencies installs Prometheus and cert-manager once for all tests +// This should be called in BeforeSuite +func InstallSharedDependencies() { + if globalDeps != nil { + GinkgoWriter.Println("⚠️ Shared dependencies already installed, skipping...") + return + } + + start := time.Now() + globalDeps = &SharedDependencies{} + + By("installing Prometheus Operator (shared)") + err := utils.InstallPrometheusOperator() + if err != nil { + // Ignore AlreadyExists errors - dependencies might be already installed + GinkgoWriter.Printf("⚠️ Prometheus Operator installation returned error (might already exist): %v\n", err) + } + globalDeps.prometheusInstalled = true + + By("installing cert-manager (shared)") + err = utils.InstallCertManager() + if err != nil { + // Ignore AlreadyExists errors - dependencies might be already installed + GinkgoWriter.Printf("⚠️ cert-manager installation returned error (might already exist): %v\n", err) + } + globalDeps.certManagerInstalled = true + + globalDeps.installTime = time.Since(start) + GinkgoWriter.Printf("✅ Shared dependencies installed in %v\n", globalDeps.installTime) +} + +// UninstallSharedDependencies removes Prometheus and cert-manager +// This should be called in AfterSuite +func UninstallSharedDependencies() { + if globalDeps == nil { + return + } + + By("uninstalling Prometheus Operator (shared)") + if globalDeps.prometheusInstalled { + utils.UninstallPrometheusOperator() + } + + By("uninstalling cert-manager (shared)") + if globalDeps.certManagerInstalled { + utils.UninstallCertManager() + } + + GinkgoWriter.Println("✅ Shared dependencies uninstalled") + globalDeps = nil +} + +// AreSharedDependenciesInstalled checks if shared dependencies are available +func AreSharedDependenciesInstalled() bool { + return globalDeps != nil && globalDeps.prometheusInstalled && globalDeps.certManagerInstalled +} diff --git a/test/e2e/framework/recorder/recorder.go b/test/e2e/framework/recorder/recorder.go new file mode 100644 index 00000000..72de48b1 --- /dev/null +++ b/test/e2e/framework/recorder/recorder.go @@ -0,0 +1,258 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package recorder + +import ( + "fmt" + "strings" + "time" +) + +// TestRecorder records test operations for reproducibility and documentation +type TestRecorder struct { + testName string + namespace string + steps []TestStep + startTime time.Time + stepOrder int +} + +// TestStep represents a single operation in a test +type TestStep struct { + Order int + Command string // Exact kubectl or shell command + Description string // Human-readable description + Input string // YAML or other input data + Expected string // Expected result + WaitFor string // Wait condition (e.g., "condition=available") + Timeout string // Timeout for the operation +} + +// NewTestRecorder creates a new test recorder +func NewTestRecorder(namespace string) *TestRecorder { + return &TestRecorder{ + namespace: namespace, + steps: make([]TestStep, 0), + startTime: time.Now(), + stepOrder: 0, + } +} + +// SetTestName sets the test name for this recording +func (r *TestRecorder) SetTestName(name string) { + r.testName = name +} + +// RecordStep records a test step +func (r *TestRecorder) RecordStep(step TestStep) { + r.stepOrder++ + step.Order = r.stepOrder + r.steps = append(r.steps, step) +} + +// GetSteps returns all recorded steps +func (r *TestRecorder) GetSteps() []TestStep { + return r.steps +} + +// ExportAsShellScript exports the recorded steps as an executable shell script +func (r *TestRecorder) ExportAsShellScript() string { + var sb strings.Builder + + // Script header + sb.WriteString("#!/bin/bash\n") + sb.WriteString("# E2E Test Playbook\n") + sb.WriteString(fmt.Sprintf("# Test: %s\n", r.testName)) + sb.WriteString(fmt.Sprintf("# Namespace: %s\n", r.namespace)) + sb.WriteString(fmt.Sprintf("# Generated: %s\n\n", time.Now().Format(time.RFC3339))) + + // Shell settings for safety + sb.WriteString("set -e # Exit on error\n") + sb.WriteString("set -u # Exit on undefined variable\n") + sb.WriteString("set -o pipefail # Catch errors in pipes\n\n") + + // Variables + sb.WriteString(fmt.Sprintf("NAMESPACE='%s'\n", r.namespace)) + sb.WriteString("KUBECTL='kubectl'\n") + sb.WriteString("TMPDIR=$(mktemp -d)\n") + sb.WriteString("trap 'rm -rf $TMPDIR' EXIT\n\n") + + // Helper functions + sb.WriteString(r.generateHelperFunctions()) + + // Main steps + sb.WriteString("# Test Steps\n") + sb.WriteString("echo '═══════════════════════════════════════════════════════════'\n") + sb.WriteString(fmt.Sprintf("echo 'Test: %s'\n", r.testName)) + sb.WriteString("echo '═══════════════════════════════════════════════════════════'\n\n") + + for _, step := range r.steps { + sb.WriteString(fmt.Sprintf("# Step %d: %s\n", step.Order, step.Description)) + sb.WriteString("echo '───────────────────────────────────────────────────────────'\n") + sb.WriteString(fmt.Sprintf("log_info 'Step %d: %s'\n", step.Order, step.Description)) + sb.WriteString("echo '───────────────────────────────────────────────────────────'\n") + + // If there's input data, save it to a temporary file + if step.Input != "" { + tmpFile := fmt.Sprintf("$TMPDIR/step-%d.yaml", step.Order) + sb.WriteString(fmt.Sprintf("cat <<'EOF' > %s\n", tmpFile)) + sb.WriteString(step.Input) + sb.WriteString("\nEOF\n") + + // Modify command to use the temp file + if strings.Contains(step.Command, "kubectl apply -f -") { + modifiedCmd := strings.Replace(step.Command, "kubectl apply -f -", fmt.Sprintf("kubectl apply -f %s", tmpFile), 1) + sb.WriteString(modifiedCmd + "\n") + } else { + sb.WriteString(step.Command + "\n") + } + } else { + sb.WriteString(step.Command + "\n") + } + + // Add expected result as comment + if step.Expected != "" { + sb.WriteString(fmt.Sprintf("# Expected: %s\n", step.Expected)) + } + + // Add wait condition if specified + if step.WaitFor != "" { + sb.WriteString(fmt.Sprintf("# Wait for: %s (timeout: %s)\n", step.WaitFor, step.Timeout)) + } + + sb.WriteString("\n") + } + + // Success message + sb.WriteString("echo '═══════════════════════════════════════════════════════════'\n") + sb.WriteString("log_success 'Test completed successfully!'\n") + sb.WriteString("echo '═══════════════════════════════════════════════════════════'\n") + + return sb.String() +} + +// ExportAsMarkdown exports the recorded steps as Markdown documentation +func (r *TestRecorder) ExportAsMarkdown() string { + var sb strings.Builder + + // Document header + sb.WriteString(fmt.Sprintf("# Test Plan: %s\n\n", r.testName)) + sb.WriteString(fmt.Sprintf("**Generated**: %s\n\n", time.Now().Format(time.RFC3339))) + sb.WriteString(fmt.Sprintf("**Namespace**: `%s`\n\n", r.namespace)) + + // Prerequisites + sb.WriteString("## Prerequisites\n\n") + sb.WriteString("- Kubernetes cluster with Vector Operator installed\n") + sb.WriteString("- kubectl configured with cluster access\n") + sb.WriteString("- Appropriate RBAC permissions\n\n") + + // Test steps + sb.WriteString("## Test Steps\n\n") + + for _, step := range r.steps { + sb.WriteString(fmt.Sprintf("### Step %d: %s\n\n", step.Order, step.Description)) + + // Command + sb.WriteString("**Command**:\n") + sb.WriteString("```bash\n") + sb.WriteString(step.Command + "\n") + sb.WriteString("```\n\n") + + // Input YAML if present + if step.Input != "" { + sb.WriteString("**Input YAML**:\n") + sb.WriteString("```yaml\n") + sb.WriteString(step.Input + "\n") + sb.WriteString("```\n\n") + } + + // Wait condition if present + if step.WaitFor != "" { + sb.WriteString(fmt.Sprintf("**Wait Condition**: `%s`\n\n", step.WaitFor)) + if step.Timeout != "" { + sb.WriteString(fmt.Sprintf("**Timeout**: %s\n\n", step.Timeout)) + } + } + + // Expected result + if step.Expected != "" { + sb.WriteString(fmt.Sprintf("**Expected Result**: %s\n\n", step.Expected)) + } + + sb.WriteString("---\n\n") + } + + return sb.String() +} + +// generateHelperFunctions generates helper shell functions for the script +func (r *TestRecorder) generateHelperFunctions() string { + return `# Helper Functions +log_info() { + echo "[INFO] $(date '+%Y-%m-%d %H:%M:%S') - $1" +} + +log_error() { + echo "[ERROR] $(date '+%Y-%m-%d %H:%M:%S') - $1" >&2 +} + +log_success() { + echo "[SUCCESS] $(date '+%Y-%m-%d %H:%M:%S') - $1" +} + +check_deployment() { + local name=$1 + local namespace=${2:-$NAMESPACE} + log_info "Checking deployment $name in namespace $namespace..." + kubectl get deployment "$name" -n "$namespace" &>/dev/null || { + log_error "Deployment $name not found!" + return 1 + } + log_info "Deployment $name exists" +} + +check_service() { + local name=$1 + local namespace=${2:-$NAMESPACE} + log_info "Checking service $name in namespace $namespace..." + kubectl get service "$name" -n "$namespace" &>/dev/null || { + log_error "Service $name not found!" + return 1 + } + log_info "Service $name exists" +} + +wait_for_pods() { + local label=$1 + local namespace=${2:-$NAMESPACE} + local timeout=${3:-120s} + log_info "Waiting for pods with label $label in namespace $namespace..." + kubectl wait --for=condition=Ready pods -l "$label" -n "$namespace" --timeout="$timeout" || { + log_error "Pods with label $label did not become ready within $timeout" + return 1 + } + log_info "Pods are ready" +} + +` +} + +// Clear clears all recorded steps +func (r *TestRecorder) Clear() { + r.steps = make([]TestStep, 0) + r.stepOrder = 0 +} diff --git a/test/e2e/framework/resources.go b/test/e2e/framework/resources.go new file mode 100644 index 00000000..9e02fe13 --- /dev/null +++ b/test/e2e/framework/resources.go @@ -0,0 +1,36 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "github.com/kaasops/vector-operator/test/e2e/framework/assertions" +) + +// Pipeline returns a pipeline resource wrapper for custom matchers +func (f *Framework) Pipeline(name string) *assertions.PipelineResource { + return assertions.NewPipelineResource(f.namespace, name) +} + +// ClusterPipeline returns a cluster-scoped pipeline resource wrapper for custom matchers +func (f *Framework) ClusterPipeline(name string) *assertions.PipelineResource { + return assertions.NewPipelineResource("", name) +} + +// Service returns a service resource wrapper for custom matchers +func (f *Framework) Service(name string) *assertions.ServiceResource { + return assertions.NewServiceResource(f.namespace, name) +} diff --git a/test/e2e/normal_mode_e2e_test.go b/test/e2e/normal_mode_e2e_test.go new file mode 100644 index 00000000..1f250574 --- /dev/null +++ b/test/e2e/normal_mode_e2e_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/kaasops/vector-operator/test/e2e/framework" + "github.com/kaasops/vector-operator/test/e2e/framework/config" +) + +// Normal Mode tests verify that the operator works correctly for standard pipelines. +// These tests cover basic Vector Agent and Aggregator functionality. +var _ = Describe("Normal Mode", Label(config.LabelSmoke, config.LabelFast), Ordered, func() { + f := framework.NewUniqueFramework("test-normal-mode") + + BeforeAll(func() { + f.Setup() + }) + + AfterAll(func() { + f.Teardown() + f.PrintMetrics() + }) + + Context("VectorPipeline basics", func() { + It("should create and validate a basic pipeline with agent", func() { + By("deploying Vector Agent") + f.ApplyTestData("normal-mode/agent.yaml") + + // Give controller time to process Vector CR Create event and create daemonset + // Normal mode requires slightly more time as it involves more resources + time.Sleep(5 * time.Second) + + By("creating a VectorPipeline") + f.ApplyTestData("normal-mode/pipeline-basic.yaml") + + By("waiting for pipeline to become valid") + f.WaitForPipelineValid("basic-pipeline") + + By("verifying agent processes the pipeline configuration") + Eventually(func() error { + // Check that agent config contains the pipeline components + return f.VerifyAgentHasPipeline("normal-agent", "basic-pipeline") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + }) + + It("should handle pipeline with transforms and multiple sinks", func() { + By("creating a complex pipeline with transforms") + f.ApplyTestData("normal-mode/pipeline-complex.yaml") + + By("waiting for pipeline to become valid") + f.WaitForPipelineValid("complex-pipeline") + + By("verifying pipeline has expected components") + // Pipeline should have sources, transforms, and sinks all in agent + Eventually(func() error { + return f.VerifyAgentHasPipeline("normal-agent", "complex-pipeline") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + }) + }) + + Context("VectorAggregator basics", func() { + It("should deploy aggregator and process pipelines", func() { + By("deploying VectorAggregator") + f.ApplyTestData("normal-mode/aggregator.yaml") + f.WaitForDeploymentReady("normal-aggregator-aggregator") + + By("creating a pipeline with aggregator role") + f.ApplyTestData("normal-mode/pipeline-aggregator-role.yaml") + + By("waiting for pipeline to become valid") + f.WaitForPipelineValid("aggregator-pipeline") + + By("verifying pipeline has aggregator role") + role := f.GetPipelineStatus("aggregator-pipeline", "role") + Expect(role).To(Equal("aggregator")) + + By("verifying aggregator processes the pipeline") + Eventually(func() error { + return f.VerifyAggregatorHasPipeline("normal-aggregator", "aggregator-pipeline") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + }) + }) + + Context("Multiple pipelines in normal mode", func() { + It("should handle multiple pipelines without conflicts", func() { + By("creating 3 pipelines in normal mode") + for i := 1; i <= 3; i++ { + f.ApplyTestDataWithVars("normal-mode/pipeline-template.yaml", + map[string]string{"{{INDEX}}": fmt.Sprintf("pipeline-%d", i)}) + } + + By("waiting for all pipelines to become valid") + f.WaitForPipelineValid("pipeline-1") + f.WaitForPipelineValid("pipeline-2") + f.WaitForPipelineValid("pipeline-3") + + By("verifying all pipelines are in agent configuration") + Eventually(func() error { + if err := f.VerifyAgentHasPipeline("normal-agent", "pipeline-1"); err != nil { + return err + } + if err := f.VerifyAgentHasPipeline("normal-agent", "pipeline-2"); err != nil { + return err + } + return f.VerifyAgentHasPipeline("normal-agent", "pipeline-3") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + }) + }) + + Context("Pipeline deletion in normal mode", func() { + It("should clean up pipeline from agent config when deleted", func() { + By("creating a pipeline") + f.ApplyTestData("normal-mode/pipeline-deletable.yaml") + f.WaitForPipelineValid("deletable-pipeline") + + By("verifying pipeline is in agent config") + Eventually(func() error { + return f.VerifyAgentHasPipeline("normal-agent", "deletable-pipeline") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + + By("deleting the pipeline") + f.DeleteResource("vectorpipeline", "deletable-pipeline") + + By("verifying pipeline is removed from agent config") + Eventually(func() bool { + err := f.VerifyAgentHasPipeline("normal-agent", "deletable-pipeline") + return err != nil // Should return error when pipeline not found + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(BeTrue()) + }) + }) + + Context("Kubernetes logs source with label selectors", func() { + It("should collect logs from pods matching label selector", func() { + By("deploying a test pod with specific labels") + f.ApplyTestData("normal-mode/test-app-pod.yaml") + f.WaitForPodReady("test-app") + + By("creating pipeline with kubernetes_logs source and label selector") + f.ApplyTestData("normal-mode/pipeline-kubernetes-logs.yaml") + f.WaitForPipelineValid("k8s-logs-pipeline") + + By("verifying agent has kubernetes_logs source") + Eventually(func() error { + return f.VerifyAgentHasPipeline("normal-agent", "k8s-logs-pipeline") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + + By("verifying pipeline role is Agent") + role := f.GetPipelineStatus("k8s-logs-pipeline", "role") + Expect(role).To(Equal("agent"), "kubernetes_logs pipeline should have agent role") + }) + }) + + Context("Namespace isolation", func() { + It("should only collect logs from the pipeline's namespace", func() { + By("creating a separate namespace") + f.ApplyTestDataWithoutNamespaceReplacement("normal-mode/namespace-isolation-ns.yaml") + + By("deploying Vector agent in isolated namespace") + // Note: In real scenario, the same Vector DaemonSet serves all namespaces + // But pipelines are namespace-scoped + + By("deploying pods in both namespaces") + f.ApplyTestData("normal-mode/namespace-isolation-pod-main.yaml") + f.ApplyTestDataWithoutNamespaceReplacement("normal-mode/namespace-isolation-pod-isolated.yaml") + f.WaitForPodReady("main-namespace-pod") + f.WaitForPodReadyInNamespace("isolated-pod", "test-normal-mode-isolated") + + By("creating pipeline in isolated namespace") + f.ApplyTestDataWithoutNamespaceReplacement("normal-mode/namespace-isolation-pipeline.yaml") + f.WaitForPipelineValidInNamespace("isolated-pipeline", "test-normal-mode-isolated") + + By("verifying namespace isolation in configuration") + // The agent config should have extra_namespace_label_selector set to the pipeline's namespace + Eventually(func() error { + return f.VerifyAgentHasPipelineInNamespace("normal-agent", "isolated-pipeline", "test-normal-mode-isolated") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + }) + }) + + Context("ClusterVectorPipeline", func() { + It("should collect logs from multiple namespaces", func() { + By("creating ClusterVectorPipeline") + f.ApplyTestDataWithoutNamespaceReplacement("normal-mode/cluster-pipeline.yaml") + f.WaitForClusterPipelineValid("cluster-wide-pipeline") + + By("deploying test pods in different namespaces with matching labels") + f.ApplyTestData("normal-mode/cluster-pipeline-pod-ns1.yaml") + f.ApplyTestDataWithoutNamespaceReplacement("normal-mode/cluster-pipeline-pod-ns2.yaml") + f.WaitForPodReady("cluster-monitored-pod-1") + f.WaitForPodReadyInNamespace("cluster-monitored-pod-2", "test-normal-mode-isolated") + + By("verifying agent processes the ClusterVectorPipeline") + Eventually(func() error { + return f.VerifyAgentHasClusterPipeline("normal-agent", "cluster-wide-pipeline") + }, config.ServiceCreateTimeout, config.DefaultPollInterval).Should(Succeed()) + + By("verifying pipeline role is Agent") + role := f.GetClusterPipelineStatus("cluster-wide-pipeline", "role") + Expect(role).To(Equal("agent"), "ClusterVectorPipeline with kubernetes_logs should have agent role") + }) + }) +}) diff --git a/test/e2e/scripts/README.md b/test/e2e/scripts/README.md new file mode 100644 index 00000000..1669ae19 --- /dev/null +++ b/test/e2e/scripts/README.md @@ -0,0 +1,40 @@ +# E2E Test Scripts + +Utilities for working with e2e test results and test environment. + +## Available Scripts + +### generate_report.py + +Generates an interactive HTML pivot grid report from e2e test results. + +**Usage:** +```bash +# From project root +make test-report + +# Or directly +cd test/e2e/results +python3 ../scripts/generate_report.py +``` + +**What it does:** +- Scans all `run-*` directories in `test/e2e/results/` +- Parses test metadata and results from each run +- Generates `test_results_report.html` with interactive pivot grid +- Shows test stability across multiple runs (flaky tests, always-failing tests, etc.) + +**Requirements:** +- Python 3.6+ +- Test results in `test/e2e/results/run-YYYY-MM-DD-HHMMSS/` format + +**Output:** +- `test/e2e/results/test_results_report.html` - Interactive HTML report + +## Adding New Scripts + +When adding new test utilities: +1. Place the script in this directory +2. Update this README with usage instructions +3. Add a Makefile target if appropriate (see `make help`) +4. Ensure the script has proper error handling and help text diff --git a/test/e2e/scripts/generate_report.py b/test/e2e/scripts/generate_report.py new file mode 100644 index 00000000..d02c1bce --- /dev/null +++ b/test/e2e/scripts/generate_report.py @@ -0,0 +1,4554 @@ +#!/usr/bin/env python3 +""" +Generate HTML Pivot Grid Report for E2E Test Results (Enhanced V2) + +Features: +- Interactive pivot grid showing test results across multiple runs +- Trend analysis charts (Pass Rate, Duration) +- Advanced Log Viewer with ANSI support and filtering +- Deep Flakiness Analysis (Score, Patterns) +- Run Comparison (New Failures, Fixed Tests) +- Smart artifact matching +- Filtering and Search +""" + +import json +import sys +import html +import re +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Any, Optional, Set +from dataclasses import dataclass, field + +# --- SVG Icons --- + +def svg_icon(name: str, size: int = 16, color: str = 'currentColor') -> str: + """Generate inline SVG icons""" + icons = { + 'search': f'', + 'copy': f'', + 'download': f'', + 'error': f'', + 'warning': f'', + 'info': f'', + 'bug': f'', + 'chevron-up': f'', + 'chevron-down': f'', + 'arrow-up': f'', + 'arrow-down': f'', + 'wrap': f'', + 'sun': f'', + 'moon': f'', + } + return icons.get(name, '') + +def format_duration(seconds: float) -> str: + """Format duration in human-readable format""" + if not seconds or seconds < 0: + return 'N/A' + + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + secs = int(seconds % 60) + + if hours > 0: + return f"{hours}h {minutes}m {secs}s" + elif minutes > 0: + return f"{minutes}m {secs}s" + else: + return f"{secs}s" + +# --- Data Structures --- + +@dataclass +class TestResult: + name: str + full_name: str + leaf_text: str + state: str + runtime: float + failure_message: str = "" + labels: List[str] = field(default_factory=list) + container_hierarchy: List[str] = field(default_factory=list) + start_time: str = "" + end_time: str = "" + artifact_metadata: Optional[Dict[str, Any]] = None + +@dataclass +class TestRun: + run_id: str + start_time: str + total_tests: int + passed_tests: int + failed_tests: int + environment: Dict[str, Any] + total_runtime: float + test_output_log: str + tests: List[TestResult] + git_commit: str = "" + git_branch: str = "" + git_dirty: str = "" + description: str = "" + + @property + def date_str(self) -> str: + return datetime.fromisoformat(self.start_time).strftime('%Y-%m-%d %H:%M') + +@dataclass +class PivotRow: + test_name: str + full_test_name: str + leaf_text: str + container_hierarchy: List[str] + runs: Dict[str, Any] = field(default_factory=dict) # run_id -> result dict + + # Stats + total_runs: int = 0 + pass_count: int = 0 + fail_count: int = 0 + skip_count: int = 0 + pass_rate: float = 0.0 + total_runtime: float = 0.0 + avg_runtime: float = 0.0 + min_runtime: float = float('inf') + max_runtime: float = 0.0 + + # Flakiness + is_flaky: bool = False + flakiness_score: float = 0.0 + flakiness_pattern: str = 'stable' + +# --- Templates --- + +class ReportTemplates: + """Holds HTML, CSS, and JS templates.""" + + CSS = """ + :root { + /* Light theme colors */ + --bg-primary: #ffffff; + --bg-secondary: #f8fafc; + --bg-tertiary: #fafbfc; + --bg-hover: #e0e7ff; + --cell-hover-bg: #eff6ff; + --cell-active-bg: #dbeafe; + --text-primary: #0f172a; + --text-secondary: #64748b; + --text-tertiary: #94a3b8; + --border-color: #e2e8f0; + --border-secondary: #cbd5e1; + --modal-backdrop: rgba(0,0,0,0.4); + --accent-color: #3b82f6; + + /* Status colors */ + --success-bg: #d1fae5; + --success-text: #065f46; + --error-bg: #fee2e2; + --error-text: #991b1b; + --warning-bg: #fef3c7; + --warning-text: #92400e; + --info-bg: #dbeafe; + --info-text: #1e40af; + + /* Log viewer colors */ + --log-bg: #1e293b; + --log-text: #e2e8f0; + --log-border: #334155; + --log-controls-bg: #334155; + --log-input-bg: #1e293b; + --log-input-border: #475569; + } + + [data-theme="dark"] { + /* Dark theme colors */ + --bg-primary: #1e293b; + --bg-secondary: #0f172a; + --bg-tertiary: #1e293b; + --bg-hover: #334155; + --cell-hover-bg: rgba(59, 130, 246, 0.2); + --cell-active-bg: rgba(59, 130, 246, 0.3); + --text-primary: #f1f5f9; + --text-secondary: #cbd5e1; + --text-tertiary: #94a3b8; + --border-color: #334155; + --border-secondary: #475569; + --modal-backdrop: rgba(0,0,0,0.7); + --accent-color: #60a5fa; + + /* Status colors */ + --success-bg: #064e3b; + --success-text: #a7f3d0; + --error-bg: #7f1d1d; + --error-text: #fca5a5; + --warning-bg: #78350f; + --warning-text: #fcd34d; + --info-bg: #1e3a8a; + --info-text: #93c5fd; + + /* Log viewer colors */ + --log-bg: #0f172a; + --log-text: #e2e8f0; + --log-border: #1e293b; + } + + /* Row hover effect */ + tbody tr { + transition: background-color 0.15s ease; + } + tbody tr:hover { + background-color: var(--bg-hover); + } + tbody tr:hover .test-name-cell, + tbody tr:hover .stats-cell { + background-color: var(--bg-hover); + } + + * { margin: 0; padding: 0; box-sizing: border-box; } + body { + font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + background: var(--bg-secondary); + color: var(--text-primary); + min-height: 100vh; + } + + /* SVG Icons */ + svg { + display: inline-block; + vertical-align: middle; + flex-shrink: 0; + } + button svg, .filter-label svg, .log-actions-item svg { + margin-right: 6px; + } + + .container { background: var(--bg-primary); min-height: 100vh; display: flex; flex-direction: column; } + + /* Header & Tabs */ + .header { + padding: 20px 40px; + border-bottom: 1px solid var(--border-color); + display: flex; + justify-content: space-between; + align-items: center; + background: var(--bg-primary); + } + .header h1 { font-size: 22px; font-weight: 600; margin-bottom: 4px; } + + .tabs { + display: flex; + padding: 0 40px; + background: var(--bg-primary); + border-bottom: 1px solid var(--border-color); + gap: 24px; + } + .tab-btn { + padding: 16px 4px; + background: none; + border: none; + border-bottom: 2px solid transparent; + color: var(--text-secondary); + font-weight: 500; + cursor: pointer; + transition: all 0.2s; + } + .tab-btn:hover { color: var(--text-primary); } + .tab-btn.active { + color: var(--accent-color); + border-bottom-color: var(--accent-color); + } + + .tab-content { display: none; padding: 24px 40px; } + .tab-content.active { display: block; } + + /* Summary Cards */ + .summary-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 16px; + margin-bottom: 24px; + } + .card { + background: var(--bg-primary); + padding: 20px; + border-radius: 8px; + border: 1px solid var(--border-color); + text-align: center; + display: flex; + flex-direction: column; + justify-content: center; + min-height: 100px; + } + .card .label { font-size: 13px; color: var(--text-secondary); text-transform: uppercase; font-weight: 600; margin-bottom: 8px; } + .card .value { font-size: 32px; font-weight: 700; color: var(--text-primary); text-align: center; display: block; } + .card.passed .value { color: #10b981; } + .card.failed .value { color: #ef4444; } + + /* Tooltips */ + .tooltip { + position: relative; + cursor: help; + } + .card.tooltip { + display: flex; + flex-direction: column; + justify-content: center; + min-height: 100px; + } + .pass-rate.tooltip { + display: inline-block; + } + .badge.tooltip { + position: relative; + display: inline-block; + } + .tooltip .tooltiptext { + visibility: hidden; + width: 250px; + background-color: #1f2937; + color: #fff; + text-align: left; + border-radius: 6px; + padding: 8px 12px; + position: absolute; + z-index: 1000; + top: 100%; + margin-top: 8px; + left: 50%; + margin-left: -125px; + opacity: 0; + transition: opacity 0.3s; + font-size: 12px; + line-height: 1.4; + box-shadow: 0 4px 6px rgba(0, 0, 0, 0.3); + pointer-events: none; + } + .tooltip .tooltiptext::after { + content: ""; + position: absolute; + bottom: 100%; + left: 50%; + margin-left: -5px; + border-width: 5px; + border-style: solid; + border-color: transparent transparent #1f2937 transparent; + } + .tooltip:hover .tooltiptext { + visibility: visible; + opacity: 1; + } + + /* Charts */ + .charts-container { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 24px; + margin-bottom: 32px; + } + .chart-wrapper { + background: var(--bg-primary); + padding: 20px; + border-radius: 12px; + border: 1px solid var(--border-color); + height: 350px; + } + + /* Flaky Section */ + .flaky-grid { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(300px, 1fr)); + gap: 16px; + margin-top: 16px; + } + .flaky-card { + text-align: left; + cursor: pointer; + transition: transform 0.2s; + } + .flaky-card:hover { transform: translateY(-2px); border-color: var(--accent-color); } + .flaky-header { display: flex; justify-content: space-between; margin-bottom: 8px; } + .flaky-name { font-weight: 500; margin-bottom: 8px; font-size: 14px; overflow: hidden; text-overflow: ellipsis; } + .flaky-stats { font-size: 12px; color: var(--text-secondary); } + + /* Table Styles */ + .table-wrapper { + overflow-x: auto; + border: 1px solid var(--border-color); + border-radius: 8px; + } + table { width: 100%; border-collapse: collapse; font-size: 14px; } + th, td { + padding: 12px 16px; + border-bottom: 1px solid var(--border-color); + text-align: left; + white-space: nowrap; + } + th { + background: var(--bg-secondary); + font-weight: 600; + color: var(--text-secondary); + position: sticky; + top: 0; + z-index: 10; + } + th.run-header { + cursor: pointer; + transition: all 0.2s ease; + } + th.run-header:hover { + background: var(--bg-hover); + color: var(--text-primary); + transform: scale(1.02); + } + th:first-child, td:first-child { + position: sticky; + left: 0; + background: var(--bg-primary); + z-index: 11; + border-right: 1px solid var(--border-color); + min-width: 500px; + max-width: 700px; + white-space: normal; + } + th:first-child { background: var(--bg-secondary); z-index: 12; } + + /* Breadcrumb Test Names */ + .test-name-cell { + padding: 10px 12px !important; + line-height: 1.5; + position: relative; + } + .test-name-wrapper { + display: flex; + align-items: center; + gap: 8px; + } + .copy-test-name-btn { + opacity: 0; + transition: opacity 0.2s ease; + background: var(--bg-primary); + border: 1px solid var(--border-color); + border-radius: 4px; + padding: 4px 8px; + cursor: pointer; + font-size: 11px; + color: var(--text-secondary); + display: flex; + align-items: center; + gap: 4px; + white-space: nowrap; + } + .copy-test-name-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + border-color: var(--text-tertiary); + } + .copy-test-name-btn:active { + transform: scale(0.95); + } + tbody tr:hover .copy-test-name-btn { + opacity: 1; + } + .test-breadcrumb { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 6px; + font-size: 13px; + } + .breadcrumb-item { + display: inline-flex; + align-items: center; + } + .breadcrumb-container { + color: var(--text-secondary); + font-weight: 500; + } + .breadcrumb-container.level-0 { + color: var(--text-primary); + font-weight: 600; + } + .breadcrumb-separator { + color: var(--text-tertiary); + margin: 0 6px; + font-weight: 300; + user-select: none; + } + .breadcrumb-leaf { + color: var(--text-primary); + font-weight: 400; + } + + .result-cell { + text-align: center; + cursor: pointer !important; + transition: all 0.2s ease; + position: relative; + } + .result-cell:hover { + background: var(--cell-hover-bg) !important; + transform: translateY(-1px); + box-shadow: 0 2px 4px rgba(0,0,0,0.1); + } + .result-cell:active { + transform: translateY(0); + } + .result-cell * { + cursor: pointer !important; + pointer-events: none; + } + .badge { + padding: 4px 8px; + border-radius: 4px; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + pointer-events: none; + } + .badge.passed { background: var(--success-bg); color: var(--success-text); } + .badge.failed { background: var(--error-bg); color: var(--error-text); } + .badge.skipped { background: var(--warning-bg); color: var(--warning-text); } + + /* Runtime Badges */ + .runtime { margin-left: 6px; font-size: 11px; pointer-events: none; } + .runtime-fast { color: #10b981; } + .runtime-medium { color: #f59e0b; } + .runtime-slow { color: #ef4444; } + + /* Stats Column */ + .stats-col, .stats-cell { + position: sticky; + left: 500px; /* after test name */ + background: var(--bg-primary); + border-right: 1px solid var(--border-color); + min-width: 120px; + text-align: center; + font-size: 12px; + z-index: 11; + } + .stats-col { z-index: 12; background: var(--bg-secondary); } + .pass-rate { padding: 4px 8px; border-radius: 4px; margin-bottom: 4px; display: inline-block; font-weight: bold; } + .rate-high { background: var(--success-bg); color: var(--success-text); } + .rate-medium { background: var(--warning-bg); color: var(--warning-text); } + .rate-low { background: var(--error-bg); color: var(--error-text); } + .counts { color: var(--text-secondary); margin-bottom: 2px; } + .avg-time { color: var(--text-tertiary); font-size: 10px; } + + /* Filters */ + .filters { + display: flex; + gap: 16px; + margin-bottom: 20px; + flex-wrap: wrap; + } + .filter-input { + padding: 8px 12px; + border: 1px solid var(--border-secondary); + border-radius: 6px; + background: var(--bg-primary); + color: var(--text-primary); + min-width: 200px; + } + + /* Modal */ + .modal { + display: none; + position: fixed; + top: 0; left: 0; width: 100%; height: 100%; + background: var(--modal-backdrop); + z-index: 1000; + backdrop-filter: blur(2px); + } + .modal-content { + background: var(--bg-primary); + width: 90%; max-width: 1200px; + margin: 20px auto 30px; + border-radius: 12px; + max-height: calc(100vh - 50px); + display: flex; + flex-direction: column; + box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.25); + } + .modal-header { + padding: 20px 30px; + border-bottom: 1px solid var(--border-color); + display: flex; + justify-content: space-between; + align-items: center; + flex-shrink: 0; + } + .modal-header h2 { + font-size: 18px; + margin: 0; + } + .modal-body { + padding: 0; + overflow-y: auto; + background: var(--bg-secondary); + flex: 1; + } + .close-btn { + font-size: 28px; + cursor: pointer; + color: var(--text-secondary); + transition: color 0.2s; + } + .close-btn:hover { + color: var(--text-primary); + } + + /* Modal Tabs */ + .modal-tabs { + display: flex; + background: var(--bg-primary); + border-bottom: 1px solid var(--border-color); + padding: 0 30px; + gap: 24px; + } + .modal-tab { + padding: 14px 4px; + background: none; + border: none; + border-bottom: 2px solid transparent; + color: var(--text-secondary); + font-weight: 500; + font-size: 14px; + cursor: pointer; + transition: all 0.2s; + } + .modal-tab:hover { + color: var(--text-primary); + } + .modal-tab.active { + color: var(--accent-color); + border-bottom-color: var(--accent-color); + } + .modal-tab-content { + display: none; + padding: 30px; + } + .modal-tab-content.active { + display: block; + } + + /* Test Detail Sections */ + .test-detail-section { + background: var(--bg-primary); + border-radius: 8px; + padding: 20px; + margin-bottom: 16px; + border: 1px solid var(--border-color); + } + .test-detail-section h4 { + margin: 0 0 16px 0; + font-size: 14px; + font-weight: 600; + color: var(--text-primary); + text-transform: uppercase; + letter-spacing: 0.5px; + } + .detail-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 16px; + } + .detail-item { + display: flex; + flex-direction: column; + } + .detail-label { + font-size: 11px; + color: var(--text-tertiary); + text-transform: uppercase; + letter-spacing: 0.5px; + margin-bottom: 4px; + } + .detail-value { + font-size: 14px; + color: var(--text-primary); + font-weight: 500; + } + .detail-value code { + background: var(--bg-secondary); + padding: 2px 6px; + border-radius: 4px; + font-size: 12px; + } + + /* Artifact List */ + .artifact-list { + display: flex; + flex-direction: column; + gap: 12px; + } + .artifact-item { + background: var(--bg-secondary); + border: 1px solid var(--border-color); + border-radius: 6px; + padding: 16px; + } + .artifact-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 12px; + } + .artifact-name { + font-weight: 600; + font-size: 14px; + color: var(--text-primary); + } + .artifact-meta { + font-size: 11px; + color: var(--text-tertiary); + } + + /* Error Box */ + .error-box { + background: var(--error-bg); + border: 1px solid var(--error-text); + border-radius: 8px; + padding: 16px; + margin-top: 16px; + } + .error-box h4 { + margin: 0 0 12px 0; + color: var(--error-text); + font-size: 14px; + } + .error-box pre { + margin: 0; + color: var(--error-text); + font-size: 12px; + white-space: pre-wrap; + word-break: break-word; + } + + /* Log Viewer */ + .log-viewer { + background: var(--log-bg); + color: var(--log-text); + padding: 0; + border-radius: 8px; + font-family: 'JetBrains Mono', monospace; + font-size: 13px; + max-height: 600px; + overflow-y: auto; + line-height: 1.6; + counter-reset: line-number; + } + .log-viewer.with-line-numbers { + padding-left: 0; + } + .log-line { + display: flex; + padding: 2px 0; + position: relative; + } + .log-line:hover { + background: rgba(59, 130, 246, 0.15); + } + .log-line-number { + counter-increment: line-number; + flex-shrink: 0; + width: 50px; + padding: 0 12px; + text-align: right; + color: var(--text-tertiary); + user-select: none; + border-right: 1px solid var(--border-color); + font-size: 11px; + line-height: 1.6; + } + .log-line-number::before { + content: counter(line-number); + } + .log-line-number:hover { + color: var(--text-secondary); + cursor: pointer; + } + .log-line-content { + flex: 1; + padding: 0 12px; + white-space: pre-wrap; + word-break: break-word; + } + + /* Log syntax highlighting */ + .log-viewer .log-passed { + color: #10b981; + font-weight: 600; + } + .log-viewer .log-failed { + color: #ef4444; + font-weight: 600; + background: rgba(239, 68, 68, 0.1); + padding: 2px 4px; + border-radius: 2px; + } + .log-viewer .log-step { + color: #3b82f6; + font-weight: 600; + } + .log-viewer .log-error { + color: #f97316; + font-weight: 600; + background: rgba(249, 115, 22, 0.1); + padding: 2px 4px; + border-radius: 2px; + } + .log-viewer .log-timestamp { + color: #6366f1; + opacity: 0.8; + font-size: 0.95em; + } + .log-viewer .log-duration { + color: #8b5cf6; + font-weight: 500; + } + .log-viewer .log-command { + color: #06b6d4; + font-style: italic; + } + .log-viewer .log-test-name { + color: #fbbf24; + font-weight: 500; + } + .log-viewer .log-warning { + color: #f59e0b; + font-weight: 500; + } + + /* Test separators in logs */ + .log-test-separator { + border-top: 2px solid var(--border-color); + margin: 20px 0 16px 0; + padding-top: 16px; + position: relative; + } + .log-test-separator::before { + content: '━━━━'; + position: absolute; + top: -13px; + left: 0; + background: var(--log-bg); + padding-right: 10px; + color: var(--border-color); + font-size: 14px; + letter-spacing: 2px; + } + .log-test-header { + font-weight: 600; + color: var(--accent-color); + font-size: 14px; + margin-bottom: 8px; + padding: 8px 12px; + background: var(--bg-hover); + border-left: 3px solid var(--accent-color); + border-radius: 4px; + } + .log-test-header .test-status { + float: right; + font-size: 12px; + padding: 2px 8px; + border-radius: 3px; + font-weight: 600; + } + .log-test-header .test-status.passed { + background: #d1fae5; + color: #065f46; + } + .log-test-header .test-status.failed { + background: #fee2e2; + color: #991b1b; + } + + .log-controls { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 12px; + padding: 10px 12px; + background: linear-gradient(to bottom, var(--bg-primary), var(--bg-hover)); + border: 1px solid var(--border-color); + border-radius: 8px; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05); + flex-wrap: wrap; + } + .log-controls > strong { + margin-right: 8px; + font-size: 13px; + color: var(--text-primary); + } + .toolbar-separator { + width: 1px; + height: 24px; + background: var(--border-color); + margin: 0 4px; + } + .log-btn { + padding: 4px 12px; + background: var(--log-controls-bg); + border: 1px solid var(--log-border); + color: var(--log-text); + border-radius: 4px; + cursor: pointer; + } + .log-btn.active { background: var(--accent-color); color: white; } + + /* Control groups */ + .log-actions-menu, + .log-filters-menu { + position: relative; + display: inline-flex; + margin-right: 6px; + } + .log-search-controls { + display: flex; + align-items: center; + gap: 4px; + padding: 4px 8px; + background: var(--bg-hover); + border-radius: 6px; + } + .log-nav-buttons { + display: flex; + gap: 4px; + padding: 4px 8px; + background: var(--bg-hover); + border-radius: 6px; + } + .log-display-controls { + display: flex; + gap: 6px; + align-items: center; + padding: 4px 8px; + background: var(--bg-hover); + border-radius: 6px; + } + .log-search-input { + padding: 4px 8px; + border: 1px solid var(--border-color); + border-radius: 4px; + font-size: 11px; + width: 180px; + outline: none; + transition: all 0.2s ease; + } + .log-search-input:focus { + border-color: var(--accent-color); + box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.1); + } + .log-search-btn { + padding: 4px 8px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + color: var(--text-secondary); + border-radius: 4px; + cursor: pointer; + font-size: 13px; + transition: all 0.2s ease; + min-width: 28px; + display: flex; + align-items: center; + justify-content: center; + } + .log-search-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + border-color: var(--accent-color); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + transform: translateY(-1px); + } + .log-search-btn:active { + transform: translateY(0); + box-shadow: none; + } + .log-search-btn:disabled { + opacity: 0.4; + cursor: not-allowed; + } + .search-counter { + font-size: 11px; + color: var(--text-secondary); + min-width: 40px; + text-align: center; + } + /* Search result highlighting */ + .search-highlight { + background: #bfdbfe; + color: #1e3a8a; + border-radius: 2px; + padding: 0 2px; + } + .theme-dark .search-highlight { + background: #1e3a8a; + color: #bfdbfe; + } + .search-highlight-current { + background: #3b82f6; + color: white; + border-radius: 2px; + padding: 0 2px; + font-weight: 600; + box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.3); + } + .theme-dark .search-highlight-current { + background: #60a5fa; + color: #0f172a; + } + + /* Actions and Filters menus */ + .log-actions-btn { + padding: 4px 8px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + color: var(--text-secondary); + border-radius: 4px; + cursor: pointer; + font-size: 16px; + transition: all 0.2s ease; + min-width: 32px; + display: flex; + align-items: center; + justify-content: center; + } + .log-actions-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + border-color: var(--accent-color); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + transform: translateY(-1px); + } + .log-actions-btn:active { + transform: translateY(0); + box-shadow: none; + } + .log-actions-dropdown { + display: none; + position: absolute; + top: 100%; + left: 0; + margin-top: 4px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + border-radius: 6px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + z-index: 1000; + min-width: 160px; + overflow: hidden; + } + .log-actions-dropdown.show { + display: block; + } + .log-actions-item { + padding: 8px 12px; + cursor: pointer; + font-size: 12px; + color: var(--text-primary); + transition: background 0.15s ease; + display: flex; + align-items: center; + gap: 8px; + } + .log-actions-item:hover { + background: var(--bg-hover); + } + .log-actions-divider { + height: 1px; + background: var(--border-color); + margin: 4px 0; + } + + /* Filters menu */ + .log-filters-btn { + padding: 4px 10px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + color: var(--text-secondary); + border-radius: 4px; + cursor: pointer; + font-size: 11px; + transition: all 0.2s ease; + display: flex; + align-items: center; + gap: 4px; + } + .log-filters-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + border-color: var(--accent-color); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1); + transform: translateY(-1px); + } + .log-filters-btn:active { + transform: translateY(0); + box-shadow: none; + } + .log-filters-dropdown { + display: none; + position: absolute; + top: 100%; + left: 0; + margin-top: 4px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + border-radius: 6px; + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + z-index: 1000; + min-width: 180px; + padding: 8px 0; + overflow: hidden; + } + .log-filters-dropdown.show { + display: block; + } + .log-filters-header { + padding: 4px 12px 8px; + font-size: 11px; + font-weight: 600; + color: var(--text-secondary); + text-transform: uppercase; + border-bottom: 1px solid var(--border-color); + margin-bottom: 4px; + } + .log-filter-item { + padding: 6px 12px; + cursor: pointer; + font-size: 12px; + color: var(--text-primary); + transition: background 0.15s ease; + display: flex; + align-items: center; + gap: 8px; + } + .log-filter-item:hover { + background: var(--bg-hover); + } + .log-filter-item input[type="checkbox"] { + margin: 0; + cursor: pointer; + } + .filter-label { + display: flex; + align-items: center; + gap: 6px; + flex: 1; + } + .filter-count { + margin-left: auto; + font-size: 11px; + color: var(--text-secondary); + background: var(--bg-hover); + padding: 2px 6px; + border-radius: 10px; + } + .log-filters-actions { + display: flex; + gap: 6px; + padding: 8px 12px 4px; + border-top: 1px solid var(--border-color); + margin-top: 4px; + } + .log-filter-action-btn { + flex: 1; + padding: 4px 8px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + color: var(--text-secondary); + border-radius: 4px; + cursor: pointer; + font-size: 11px; + transition: all 0.2s ease; + } + .log-filter-action-btn:hover { + background: var(--bg-hover); + border-color: var(--accent-color); + } + .log-filter-action-btn.primary { + background: var(--accent-color); + color: white; + border-color: var(--accent-color); + } + .log-filter-action-btn.primary:hover { + background: #2563eb; + } + + /* Navigation buttons */ + .log-nav-btn { + padding: 4px 10px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + color: var(--text-secondary); + border-radius: 4px; + cursor: pointer; + font-size: 11px; + transition: all 0.2s ease; + display: flex; + align-items: center; + gap: 4px; + } + .log-nav-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + border-color: var(--accent-color); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + transform: translateY(-1px); + } + .log-nav-btn:active { + transform: translateY(0); + box-shadow: none; + } + .log-nav-btn:disabled { + opacity: 0.4; + cursor: not-allowed; + } + .log-nav-btn:disabled:hover { + background: var(--bg-primary); + color: var(--text-secondary); + border-color: var(--border-color); + } + .log-nav-btn .nav-icon { + font-size: 12px; + } + .log-nav-btn.error-nav { + border-color: #ef4444; + color: #ef4444; + } + .log-nav-btn.error-nav:hover { + background: #fef2f2; + box-shadow: 0 1px 3px rgba(239, 68, 68, 0.2); + transform: translateY(-1px); + } + + /* Display controls */ + .log-zoom-control { + display: flex; + gap: 2px; + align-items: center; + } + .log-zoom-btn { + padding: 2px 6px; + background: var(--bg-primary); + border: 1px solid var(--border-color); + color: var(--text-secondary); + border-radius: 3px; + cursor: pointer; + font-size: 12px; + font-weight: 600; + transition: all 0.2s ease; + } + .log-zoom-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + border-color: var(--accent-color); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + transform: scale(1.05); + } + .log-zoom-btn:active { + transform: scale(0.95); + box-shadow: none; + } + .log-zoom-value { + font-size: 11px; + color: var(--text-tertiary); + min-width: 35px; + text-align: center; + } + .log-viewer.font-sm { font-size: 11px; } + .log-viewer.font-md { font-size: 13px; } + .log-viewer.font-lg { font-size: 15px; } + .log-viewer.font-xl { font-size: 17px; } + .log-viewer.wrap-enabled .log-line-content { + white-space: pre-wrap; + word-break: break-word; + } + .log-viewer.wrap-disabled .log-line-content { + white-space: pre; + overflow-x: auto; + } + + /* Floating Controls Panel (Google Maps style) */ + .log-floating-controls { + position: absolute; + bottom: 16px; + right: 16px; + display: flex; + flex-direction: column; + gap: 10px; + z-index: 100; + pointer-events: none; + opacity: 0.3; + transition: opacity 0.2s ease; + } + .log-floating-controls:hover { + opacity: 1; + } + .log-floating-controls > * { + pointer-events: auto; + } + .floating-control-group { + background: rgba(255, 255, 255, 0.95); + border: none; + border-radius: 2px; + padding: 4px; + box-shadow: rgba(0, 0, 0, 0.3) 0px 1px 4px -1px; + display: flex; + flex-direction: column; + gap: 0; + align-items: center; + } + .theme-dark .floating-control-group { + background: rgba(30, 41, 59, 0.75); + box-shadow: rgba(0, 0, 0, 0.4) 0px 2px 6px 0px; + backdrop-filter: blur(8px); + } + .floating-control-divider { + width: 20px; + height: 1px; + background: #e5e7eb; + margin: 4px 0; + } + .theme-dark .floating-control-divider { + background: #4a5568; + } + .floating-zoom-control { + display: flex; + flex-direction: column; + gap: 0; + align-items: center; + } + .floating-zoom-btn { + width: 28px; + height: 28px; + padding: 0; + background: transparent; + border: none; + color: #5f6368; + border-radius: 2px; + cursor: pointer; + font-size: 18px; + font-weight: 400; + line-height: 1; + transition: background 0.1s ease; + display: flex; + align-items: center; + justify-content: center; + } + .floating-zoom-btn:hover { + background: #f1f3f4; + } + .theme-dark .floating-zoom-btn { + color: #e2e8f0; + } + .theme-dark .floating-zoom-btn:hover { + background: #4a5568; + } + .floating-zoom-btn:active { + background: #e8eaed; + } + .theme-dark .floating-zoom-btn:active { + background: #2d3748; + } + .floating-zoom-value { + font-size: 10px; + color: #5f6368; + font-weight: 400; + text-align: center; + min-width: 28px; + padding: 2px 0; + } + .theme-dark .floating-zoom-value { + color: #cbd5e0; + } + .floating-wrap-btn { + width: 28px; + height: 28px; + padding: 0; + background: transparent; + border: none; + color: #5f6368; + border-radius: 2px; + cursor: pointer; + font-size: 14px; + line-height: 1; + transition: background 0.1s ease; + display: flex; + align-items: center; + justify-content: center; + } + .floating-wrap-btn:hover { + background: #f1f3f4; + } + .theme-dark .floating-wrap-btn { + color: #e2e8f0; + } + .theme-dark .floating-wrap-btn:hover { + background: #4a5568; + } + .floating-wrap-btn:active { + background: #e8eaed; + } + .theme-dark .floating-wrap-btn:active { + background: #2d3748; + } + .floating-wrap-btn.active { + background: #e8f0fe; + color: #1a73e8; + } + .theme-dark .floating-wrap-btn.active { + background: #3b82f6; + color: white; + } + .log-line-number-clickable { + position: relative; + cursor: pointer; + } + .log-line-number-clickable:hover { + background: rgba(59, 130, 246, 0.1); + } + + /* pprof Visualization - Compact */ + .pprof-compact-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 12px; + padding: 8px 12px; + background: var(--bg-secondary); + border-radius: 6px; + flex-wrap: wrap; + gap: 8px; + } + .pprof-stats { + display: flex; + align-items: center; + gap: 6px; + font-size: 13px; + color: var(--text-primary); + flex-wrap: wrap; + } + .pprof-stat strong { + color: #3b82f6; + } + .pprof-stat-sep { + color: var(--text-secondary); + font-size: 10px; + } + .pprof-help-btn { + display: flex; + align-items: center; + gap: 4px; + font-size: 11px; + color: var(--text-secondary); + background: transparent; + border: 1px solid var(--border-color); + padding: 4px 8px; + border-radius: 4px; + cursor: pointer; + transition: all 0.2s; + } + .pprof-help-btn:hover { + background: var(--bg-hover); + color: var(--text-primary); + } + .pprof-help-popup { + display: none; + position: absolute; + right: 0; + top: 100%; + margin-top: 4px; + width: 320px; + padding: 12px; + background: #1f2937; + color: #fff; + border-radius: 8px; + font-size: 12px; + line-height: 1.5; + box-shadow: 0 4px 12px rgba(0,0,0,0.3); + z-index: 100; + } + .pprof-help-popup.visible { + display: block; + } + .pprof-help-popup code { + background: rgba(255,255,255,0.1); + padding: 2px 6px; + border-radius: 3px; + font-size: 11px; + } + .pprof-compact-header { + position: relative; + } + .pprof-bars { + display: flex; + flex-direction: column; + gap: 4px; + } + .pprof-bar-item { + display: grid; + grid-template-columns: 150px 1fr; + gap: 8px; + align-items: center; + } + .pprof-bar-label { + font-size: 11px; + color: var(--text-secondary); + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + font-family: monospace; + } + .pprof-bar-container { + position: relative; + height: 18px; + background: var(--bg-secondary); + border-radius: 3px; + overflow: hidden; + } + .pprof-bar { + height: 100%; + border-radius: 3px; + background: linear-gradient(90deg, #3b82f6, #8b5cf6); + } + .pprof-bar-value { + position: absolute; + right: 6px; + top: 50%; + transform: translateY(-50%); + font-size: 10px; + font-weight: 600; + color: var(--text-primary); + } + .pprof-stacks { + display: flex; + flex-direction: column; + gap: 2px; + } + .pprof-stack-item { + background: var(--bg-secondary); + border-radius: 4px; + overflow: hidden; + } + .pprof-stack-header { + display: flex; + align-items: center; + padding: 6px 10px; + cursor: pointer; + gap: 8px; + transition: background 0.2s; + } + .pprof-stack-header:hover { + background: var(--bg-hover); + } + .pprof-stack-count { + background: #10b981; + color: white; + padding: 1px 6px; + border-radius: 8px; + font-size: 10px; + font-weight: 600; + min-width: 32px; + text-align: center; + } + .pprof-stack-name { + flex: 1; + font-size: 11px; + font-family: monospace; + color: var(--text-secondary); + text-overflow: ellipsis; + overflow: hidden; + white-space: nowrap; + } + .pprof-stack-toggle { + color: var(--text-secondary); + font-size: 9px; + transition: transform 0.2s; + } + .pprof-stack-item.expanded .pprof-stack-toggle { + transform: rotate(90deg); + } + .pprof-stack-frames { + display: none; + padding: 0 10px 8px 48px; + font-size: 10px; + font-family: monospace; + color: var(--text-secondary); + } + .pprof-stack-item.expanded .pprof-stack-frames { + display: block; + } + .pprof-frame { + padding: 2px 0; + border-left: 2px solid var(--border-color); + padding-left: 10px; + margin-left: 2px; + } + .pprof-section { + margin-bottom: 24px; + } + .pprof-section-header { + display: flex; + align-items: center; + justify-content: space-between; + margin-bottom: 12px; + } + .pprof-section-title { + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + font-weight: 600; + color: var(--text-primary); + } + .pprof-help-link { + font-size: 11px; + color: #3b82f6; + text-decoration: none; + display: flex; + align-items: center; + gap: 4px; + } + .pprof-help-link:hover { + text-decoration: underline; + } + .pprof-raw-toggle { + font-size: 11px; + color: var(--text-secondary); + background: var(--bg-secondary); + border: 1px solid var(--border-color); + padding: 4px 10px; + border-radius: 4px; + cursor: pointer; + margin-top: 12px; + } + .pprof-raw-toggle:hover { + background: var(--bg-hover); + } + .pprof-raw-content { + display: none; + margin-top: 12px; + } + .pprof-raw-content.visible { + display: block; + } + + /* Comparison */ + .comparison-run-info { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 20px; + margin-top: 20px; + } + .run-info-card { + background: var(--bg-primary); + padding: 15px; + border-radius: 8px; + border: 1px solid var(--border-color); + } + .run-info-card h4 { + margin: 0 0 10px 0; + color: var(--text-primary); + font-size: 14px; + } + .run-info-details { + font-size: 12px; + color: var(--text-secondary); + } + .run-info-details > div { + margin: 5px 0; + } + .comparison-grid { + display: grid; + grid-template-columns: 1fr 1fr 1fr; + gap: 20px; + margin-top: 20px; + } + .comparison-col { + background: var(--bg-primary); + padding: 20px; + border-radius: 8px; + border: 1px solid var(--border-color); + } + + /* Utility */ + .hidden { display: none !important; } + .theme-toggle { + background: var(--bg-hover); + border: 1px solid var(--border-color); + padding: 8px 12px; + border-radius: 6px; + cursor: pointer; + color: var(--text-primary); + } + + @media print { + .theme-toggle, .tabs, .filters, .log-controls, .close-btn { display: none !important; } + .container { display: block; } + .tab-content { display: block !important; padding: 0; } + .card, .chart-wrapper, .table-wrapper { border: 1px solid #ddd; break-inside: avoid; } + body { background: white; color: black; } + * { box-shadow: none !important; } + } + """ + + JS = """ + // ============================================================ + // SECTION: Icons & Utilities + // ============================================================ + + function svgIcon(name, size = 16, color = 'currentColor') { + const icons = { + 'search': ``, + 'copy': ``, + 'download': ``, + 'error': ``, + 'warning': ``, + 'info': ``, + 'bug': ``, + 'chevron-up': ``, + 'chevron-down': ``, + 'arrow-up': ``, + 'arrow-down': ``, + 'wrap': ``, + 'sun': ``, + 'moon': ``, + }; + return icons[name] || ''; + } + + // ============================================================ + // SECTION: Global State & Navigation + // ============================================================ + + let currentTab = 'dashboard'; + + function switchTab(tabId) { + document.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('active')); + document.querySelectorAll('.tab-content').forEach(c => c.classList.remove('active')); + + document.querySelector(`[onclick="switchTab('${tabId}')"]`).classList.add('active'); + document.getElementById(tabId).classList.add('active'); + currentTab = tabId; + } + + // Theme + function toggleTheme() { + const html = document.documentElement; + const current = html.getAttribute('data-theme'); + const next = current === 'dark' ? 'light' : 'dark'; + html.setAttribute('data-theme', next); + localStorage.setItem('theme', next); + updateThemeButton(); + } + + function updateThemeButton() { + const theme = document.documentElement.getAttribute('data-theme'); + const btn = document.getElementById('themeToggle'); + if (btn) { + btn.innerHTML = theme === 'dark' ? svgIcon('sun', 18) : svgIcon('moon', 18); + btn.title = theme === 'dark' ? 'Switch to light mode' : 'Switch to dark mode'; + } + } + + // Init + document.addEventListener('DOMContentLoaded', () => { + const savedTheme = localStorage.getItem('theme') || + (window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'); + document.documentElement.setAttribute('data-theme', savedTheme); + updateThemeButton(); + + renderCharts(); + }); + + // Filtering + function filterTable() { + const search = document.getElementById('searchInput').value.toLowerCase(); + const status = document.getElementById('statusFilter').value; + const stability = document.getElementById('stabilityFilter').value; + const label = document.getElementById('labelFilter').value; + + const rows = document.querySelectorAll('#resultsTable tbody tr'); + + rows.forEach(row => { + const rowData = JSON.parse(row.dataset.json); + const name = rowData.test_name.toLowerCase(); + + // Search + const matchesSearch = name.includes(search); + + // Status (check if ANY run matches) + let matchesStatus = status === 'all'; + if (!matchesStatus) { + matchesStatus = Object.values(rowData.runs).some(r => r && r.state === status); + } + + // Stability + let matchesStability = true; + if (stability === 'flaky') matchesStability = rowData.is_flaky; + if (stability === 'stable') matchesStability = !rowData.is_flaky && rowData.fail_count === 0; + if (stability === 'always-failing') matchesStability = rowData.pass_count === 0; + + // Label + let matchesLabel = label === 'all'; + if (!matchesLabel) { + // Check if any run has this label (simplified) + matchesLabel = Object.values(rowData.runs).some(r => r && r.labels && r.labels.includes(label)); + } + + if (matchesSearch && matchesStatus && matchesStability && matchesLabel) { + row.style.display = ''; + } else { + row.style.display = 'none'; + } + }); + } + + // Modal Tabs + function switchModalTab(tabId) { + // Update tab buttons + document.querySelectorAll('.modal-tab').forEach(btn => btn.classList.remove('active')); + document.querySelector(`[onclick="switchModalTab('${tabId}')"]`).classList.add('active'); + + // Update tab content + document.querySelectorAll('.modal-tab-content').forEach(content => content.classList.remove('active')); + document.getElementById(tabId).classList.add('active'); + } + + // ============================================================ + // SECTION: Modal & Test Details + // ============================================================ + + function showTestDetails(data) { + const modal = document.getElementById('testModal'); + const modalBody = document.getElementById('modalContent'); + + // Check if this is pivot data (flaky tests) or single test data + const isPivotData = data.state === undefined; + + // Build tabs - only Summary tab for pivot data + let html = ` +

+ + `; + + if (!isPivotData) { + html += ` + + + + `; + } + + html += ` +
+ `; + + // Summary Tab + html += ` +
+
+

Test Information

+
+
+
Test Name
+
${escapeHtml(data.test_name)}
+
+ `; + + // Single test data (from clicking on a result cell) + if (data.state !== undefined) { + html += ` +
+
Status
+
${data.state}
+
+
+
Runtime
+
${data.runtime ? data.runtime.toFixed(2) : 'N/A'}s
+
+
+
Run ID
+
#${data.run_id || 'N/A'}
+
+ `; + } else { + // Pivot data (from flaky tests - aggregated across runs) + html += ` +
+
Total Runs
+
${data.pass_count + data.fail_count}
+
+
+
Pass Count
+
${data.pass_count}
+
+
+
Fail Count
+
${data.fail_count}
+
+
+
Flakiness
+
${data.is_flaky ? '' + (data.flakiness_score ? data.flakiness_score.toFixed(0) + '% Stable' : 'Flaky') + (data.flakiness_pattern ? ' (' + data.flakiness_pattern + ')' : '') + '' : 'Stable'}
+
+ `; + } + + html += ` +
+
+ `; + + // For pivot data (flaky tests), show run history + if (data.state === undefined && data.runs) { + html += ` +
+

Run History

+
+ + + + + + + + + + `; + + Object.entries(data.runs).forEach(([runId, result]) => { + html += ` + + + + + + `; + }); + + html += ` + +
Run IDStatusRuntime
#${runId}${result.state}${result.runtime ? result.runtime.toFixed(2) + 's' : 'N/A'}
+
+
+ `; + } + + if (data.artifact_metadata) { + const meta = data.artifact_metadata; + html += ` +
+

Execution Details

+
+
+
Namespace
+
${meta.namespace || 'N/A'}
+
+
+
Start Time
+
${formatTimestamp(meta.start_time)}
+
+
+
End Time
+
${formatTimestamp(meta.end_time)}
+
+
+
Duration
+
${meta.duration_ms ? (meta.duration_ms / 1e9).toFixed(2) + 's' : 'N/A'}
+
+
+
+ `; + + if (meta.artifacts) { + html += ` +
+

Artifact Summary

+
+
+
Log Files
+
${meta.artifacts.log_files?.length || 0}
+
+
+
Resource Files
+
${meta.artifacts.resource_files?.length || 0}
+
+
+
Event Files
+
${meta.artifacts.event_files?.length || 0}
+
+
+
Collection Time
+
${meta.artifacts.collection_time || 'N/A'}
+
+
+
+ `; + } + } + + if (data.failure_message) { + html += ` +
+

Failure Message

+ 
${escapeHtml(data.failure_message)}
+
+ `; + } + + html += `
`; // Close summary tab + + // Queue for pprof rendering - declared at function scope for access after DOM update + let pprofRenderQueue = []; + + // Only show these tabs for single test data (not pivot data) + if (!isPivotData) { + // Artifacts Tab + html += `
`; + if (data.artifact_metadata?.relative_path) { + html += ` +
+

Artifact Location

+
${data.artifact_metadata.relative_path}
+
+ `; + } + if (data.artifact_metadata?.artifacts) { + const artifacts = data.artifact_metadata.artifacts; + if (artifacts.log_files?.length > 0) { + html += ` +
+

Log Files (${artifacts.log_files.length})

+
+ ${artifacts.log_files.map(f => `
📄 ${f}
`).join('')} +
+
+ `; + } + if (artifacts.resource_files?.length > 0) { + html += ` +
+

Resource Files (${artifacts.resource_files.length})

+
+ ${artifacts.resource_files.map(f => `
📦 ${f}
`).join('')} +
+
+ `; + } + if (artifacts.event_files?.length > 0) { + html += ` +
+

Event Files (${artifacts.event_files.length})

+
+ ${artifacts.event_files.map(f => `
📊 ${f}
`).join('')} +
+
+ `; + } + } else { + html += `

No artifacts available for this test run.

`; + } + html += `
`; // Close artifacts tab + + // Logs Tab + html += `
`; + if (data.artifact_metadata?.file_contents) { + const logFiles = Object.entries(data.artifact_metadata.file_contents) + .filter(([name, data]) => data.type === 'log'); + + if (logFiles.length > 0) { + logFiles.forEach(([filename, fileData], index) => { + const viewerId = `testLogViewer${index}`; + const navPrevId = `testNavPrevError${index}`; + const navNextId = `testNavNextError${index}`; + const zoomDisplayId = `testZoomDisplay${index}`; + const wrapBtnId = `testWrapToggleBtn${index}`; + const searchInputId = `testLogSearchInput${index}`; + const searchCounterId = `testLogSearchCounter${index}`; + + html += ` +
+
+ ${filename} + ${fileData.truncated ? `(showing last ${fileData.total_lines > 500 ? 500 : fileData.total_lines} lines of ${fileData.total_lines})` : ''} +
+
+ +
+
${svgIcon('copy', 14)} Copy All
+
${svgIcon('copy', 14)} Copy Visible
+
+
${svgIcon('download', 14)} Download .txt
+
${svgIcon('download', 14)} Download .log
+
+
+
+ +
+
Show:
+ + + + +
+ + +
+
+
+
+
+ + + + 0/0 +
+
+ + + + +
+
+
+
+ ${ansiToHtml(fileData.content)} +
+
+
+
+ + 100% + +
+
+ +
+
+
+
+ `; + }); + } else { + html += `

No log files available.

`; + } + } else { + html += `

No logs available for this test run.

`; + } + html += `
`; // Close logs tab + + // Resources Tab + html += `
`; + if (data.artifact_metadata?.file_contents) { + const resourceFiles = Object.entries(data.artifact_metadata.file_contents) + .filter(([name, data]) => data.type === 'resource' || data.type === 'events'); + + // Separate pprof files from other resources + const pprofFiles = resourceFiles.filter(([name]) => isPprofFile(name)); + const otherFiles = resourceFiles.filter(([name]) => !isPprofFile(name)); + + // Render pprof files with visualization + if (pprofFiles.length > 0) { + let pprofIdx = 0; + pprofFiles.forEach(([filename, fileData]) => { + const containerId = 'pprof-viz-' + pprofIdx++; + const rawId = 'pprof-raw-' + pprofIdx; + const isHeap = filename.includes('pprof-heap'); + const icon = isHeap ? '📊' : '🔄'; + const title = isHeap ? 'Heap Profile' : 'Goroutine Profile'; + + html += ` +
+
+
+ ${icon} + ${title} + (${filename}) +
+
+
+ +
+
${escapeHtml(fileData.content)}
+
+
+ `; + + // Queue for rendering after DOM update + pprofRenderQueue.push({ filename, content: fileData.content, containerId }); + }); + } + + // Render other resource files normally + if (otherFiles.length > 0) { + otherFiles.forEach(([filename, fileData]) => { + html += ` +
+

${filename}

+
${escapeHtml(fileData.content)}
+
+ `; + }); + } + + if (pprofFiles.length === 0 && otherFiles.length === 0) { + html += `

No resource files available.

`; + } + } else { + html += `

No resources available for this test run.

`; + } + html += `
`; // Close resources tab + } // End if (!isPivotData) + + modalBody.innerHTML = html; + + // Render pprof visualizations after DOM is updated + if (pprofRenderQueue && pprofRenderQueue.length > 0) { + pprofRenderQueue.forEach(item => { + renderPprofFile(item.filename, item.content, item.containerId); + }); + } + + // Apply syntax highlighting to all log viewers + modalBody.querySelectorAll('.log-viewer').forEach(viewer => { + viewer.innerHTML = highlightLogSyntax(viewer.innerHTML); + viewer.classList.add('with-line-numbers'); + // Initialize navigation and display controls for each log viewer + if (viewer.id) { + initLogNavigation(viewer); + initDisplayControls(viewer.id); + } + }); + + // Update modal title + document.getElementById('testModalTitle').textContent = 'Test Run Details'; + + modal.style.display = 'block'; + } + + function formatTimestamp(ts) { + if (!ts) return 'N/A'; + try { + return new Date(ts).toLocaleString(); + } catch { + return ts; + } + } + + // Format duration in human-readable format + function formatDuration(seconds) { + if (!seconds || seconds < 0) return 'N/A'; + + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const secs = Math.floor(seconds % 60); + + if (hours > 0) { + return `${hours}h ${minutes}m ${secs}s`; + } else if (minutes > 0) { + return `${minutes}m ${secs}s`; + } else { + return `${secs}s`; + } + } + + // Copy test name to clipboard + function copyTestName(btn) { + const testName = btn.getAttribute('data-test-name'); + + // Use Clipboard API + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(testName).then(() => { + // Visual feedback + const originalText = btn.innerHTML; + btn.innerHTML = '✓ Copied!'; + btn.style.background = '#10b981'; + btn.style.color = 'white'; + btn.style.borderColor = '#10b981'; + + setTimeout(() => { + btn.innerHTML = originalText; + btn.style.background = ''; + btn.style.color = ''; + btn.style.borderColor = ''; + }, 1500); + }).catch(err => { + console.error('Failed to copy:', err); + btn.innerHTML = '✗ Failed'; + setTimeout(() => { + btn.innerHTML = originalText; + }, 1500); + }); + } else { + // Fallback for older browsers + const originalText = btn.innerHTML; + const textarea = document.createElement('textarea'); + textarea.value = testName; + textarea.style.position = 'fixed'; + textarea.style.opacity = '0'; + document.body.appendChild(textarea); + textarea.select(); + try { + document.execCommand('copy'); + btn.innerHTML = '✓ Copied!'; + btn.style.background = '#10b981'; + btn.style.color = 'white'; + setTimeout(() => { + btn.innerHTML = originalText; + btn.style.background = ''; + btn.style.color = ''; + }, 1500); + } catch (err) { + console.error('Fallback copy failed:', err); + } + document.body.removeChild(textarea); + } + } + + // Copy to clipboard with visual feedback on the clicked element + function copyWithElementFeedback(text, element) { + const showSuccess = () => { + const originalHTML = element.innerHTML; + element.innerHTML = svgIcon('copy', 12) + ' Copied!'; + element.style.background = '#10b981'; + element.style.color = 'white'; + setTimeout(() => { + element.innerHTML = originalHTML; + element.style.background = ''; + element.style.color = ''; + }, 1500); + }; + + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(text).then(showSuccess).catch(err => { + console.error('Failed to copy:', err); + fallbackCopyToClipboard(text, 'Copied!'); + }); + } else { + fallbackCopyToClipboard(text, 'Copied!'); + showSuccess(); + } + } + + // Add visual separators between tests + function addTestSeparators(text) { + if (!text) return ''; + + // Split by Ginkgo test separators (lines with multiple dashes) + const lines = text.split('\\n'); + const result = []; + let inTest = false; + let testName = ''; + let testStatus = ''; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Detect test separator line + if (line.match(/^-{30,}$/)) { + // This is a Ginkgo test separator + result.push(line); + inTest = false; + continue; + } + + // Detect test name (lines that start with test description - typically after separator) + // Pattern: "Test Description" followed by optional labels like [smoke, slow] + const testNameMatch = line.match(/^([A-Z][^\\n]+?(?:should|must|can|will|does)[^\\n]*?)(?:\\s+\\[.*?\\])?$/); + if (testNameMatch && !inTest) { + testName = testNameMatch[1].trim(); + // Add separator and header + result.push('
'); + result.push(`
${testName}
`); + result.push(line); + inTest = true; + continue; + } + + // Detect test completion with status + const passedMatch = line.match(/\\[38;5;10m\\[.*?PASSED.*?\\[(\\d+\\.\\d+)\\s*seconds\\].*?\\[0m/); + const failedMatch = line.match(/\\[38;5;9m\\[.*?FAILED.*?\\[(\\d+\\.\\d+)\\s*seconds\\].*?\\[0m/); + + if (passedMatch || failedMatch) { + testStatus = passedMatch ? 'passed' : 'failed'; + const duration = passedMatch ? passedMatch[1] : (failedMatch ? failedMatch[1] : ''); + + // Update header if we have a test name + if (testName && inTest) { + const headerIndex = result.lastIndexOf(`
${testName}
`); + if (headerIndex !== -1) { + result[headerIndex] = `
${testName}${testStatus.toUpperCase()} ${duration}s
`; + } + } + inTest = false; + testName = ''; + } + + result.push(line); + } + + return result.join('\\n'); + } + + // Add line numbers to log output + function addLineNumbers(text) { + if (!text) return ''; + + const lines = text.split('\\n'); + const numberedLines = lines.map(line => { + // Skip empty lines at the end + if (line.trim() === '' && lines[lines.length - 1] === line) { + return ''; + } + return `
${line}
`; + }).filter(line => line !== ''); + + return numberedLines.join(''); + } + + // Highlight log syntax for better readability + function highlightLogSyntax(text) { + if (!text) return ''; + + // First add test separators + text = addTestSeparators(text); + + // Then apply syntax highlighting + text = text + // PASSED/passed with checkmark + .replace(/(\\[38;5;10m\\[.*?PASSED.*?\\[0m|\\bPASSED\\b|✅|passed)/gi, '$1') + // FAILED/failed with cross + .replace(/(\\[38;5;9m\\[.*?FAILED.*?\\[0m|\\bFAILED\\b|❌|✗|failed)/gi, '$1') + // STEP markers + .replace(/(\\[1mSTEP:\\[0m|STEP:)/g, '$1') + // ERROR/error + .replace(/(\\bERROR\\b|\\berror\\b|\\bError\\b|🔥)/gi, '$1') + // Warnings + .replace(/(⚠️|WARNING|warning)/gi, '$1') + // Timestamps like @ 11/19/25 15:35:34.904 + .replace(/(@ \\d{2}\\/\\d{2}\\/\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d+)/g, '$1') + // Duration like [35.588 seconds] or 2.5s + .replace(/(\\[\\d+\\.\\d+ seconds\\]|\\d+\\.\\d+s)/g, '$1') + // kubectl/running commands + .replace(/(running:|kubectl|KUBECTL_CMD:)([^\\n]*)/gi, '$1$2') + // Test names (lines starting with test description) + .replace(/^([▸●] .*$)/gm, '$1'); + + // Finally add line numbers + return addLineNumbers(text); + } + + // ============================================================ + // SECTION: Log Viewer State & Navigation + // ============================================================ + + const logViewerStates = new WeakMap(); + + function getLogViewerState(viewer) { + if (!logViewerStates.has(viewer)) { + logViewerStates.set(viewer, { + // Error navigation + currentErrorIndex: -1, + errorPositions: [], + // Display settings + currentFontIndex: 1, + wrapEnabled: true, + // Search state + searchIndex: -1, + searchMatches: [], + originalContent: null, + // Filter state + filters: { error: true, warning: true, info: true, debug: true } + }); + } + return logViewerStates.get(viewer); + } + + // Legacy accessors for compatibility during refactoring + function getViewerState(viewer) { return getLogViewerState(viewer); } + + function findErrorsInLog(viewer) { + const state = getViewerState(viewer); + state.errorPositions = []; + const uniqueLines = new Set(); + const errorElements = viewer.querySelectorAll('.log-failed, .log-error'); + errorElements.forEach(elem => { + const line = elem.closest('.log-line'); + if (line && !uniqueLines.has(line)) { + uniqueLines.add(line); + state.errorPositions.push(line); + } + }); + return state.errorPositions.length; + } + + function navigateLog(viewer, direction) { + if (direction === 'top') { + viewer.scrollTop = 0; + return; + } + if (direction === 'bottom') { + viewer.scrollTop = viewer.scrollHeight; + return; + } + + // Error navigation + const state = getViewerState(viewer); + if (state.errorPositions.length === 0) { + findErrorsInLog(viewer); + } + if (state.errorPositions.length === 0) { + showNotification('No errors found in logs', 'info'); + return; + } + + if (direction === 'next') { + state.currentErrorIndex = (state.currentErrorIndex + 1) % state.errorPositions.length; + } else if (direction === 'prev') { + state.currentErrorIndex = state.currentErrorIndex <= 0 ? state.errorPositions.length - 1 : state.currentErrorIndex - 1; + } + + scrollToElement(state.errorPositions[state.currentErrorIndex], viewer); + updateNavButtons(viewer); + } + + // Legacy wrappers for compatibility + function jumpToNextError(viewer) { navigateLog(viewer, 'next'); } + function jumpToPrevError(viewer) { navigateLog(viewer, 'prev'); } + function jumpToTop(viewer) { navigateLog(viewer, 'top'); } + function jumpToBottom(viewer) { navigateLog(viewer, 'bottom'); } + + function scrollToElement(element, container) { + if (!element || !container) return; + + // Highlight the line temporarily + const isDark = document.documentElement.classList.contains('theme-dark'); + element.style.transition = 'background 0.3s ease'; + element.style.background = isDark ? 'rgba(59, 130, 246, 0.2)' : 'rgba(147, 197, 253, 0.4)'; + + // Scroll to element + const containerRect = container.getBoundingClientRect(); + const elementRect = element.getBoundingClientRect(); + const scrollTop = container.scrollTop; + const offset = elementRect.top - containerRect.top + scrollTop - 100; + + container.scrollTo({ + top: offset, + behavior: 'smooth' + }); + + // Remove highlight after delay + setTimeout(() => { + element.style.background = ''; + }, 1500); + } + + function updateNavButtons(viewer) { + const state = getViewerState(viewer); + const errorCount = state.errorPositions.length; + + // Derive button IDs from viewer ID + const viewerId = viewer.id; + let prevBtnId, nextBtnId; + + if (viewerId.startsWith('testLogViewer')) { + // Extract index from testLogViewer0, testLogViewer1, etc. + const index = viewerId.replace('testLogViewer', ''); + prevBtnId = `testNavPrevError${index}`; + nextBtnId = `testNavNextError${index}`; + } else { + // Default for run log viewer + prevBtnId = 'navPrevError'; + nextBtnId = 'navNextError'; + } + + const prevBtn = document.getElementById(prevBtnId); + const nextBtn = document.getElementById(nextBtnId); + + if (prevBtn && nextBtn) { + const countText = errorCount > 0 ? ` (${state.currentErrorIndex + 1}/${errorCount})` : ' (0)'; + prevBtn.disabled = errorCount === 0; + nextBtn.disabled = errorCount === 0; + + // Update button text with current position + if (errorCount > 0) { + nextBtn.innerHTML = `↓ Next Error ${countText}`; + prevBtn.innerHTML = `↑ Prev Error`; + } + } + } + + function initLogNavigation(viewer) { + const errorCount = findErrorsInLog(viewer); + const state = getViewerState(viewer); + state.currentErrorIndex = -1; + updateNavButtons(viewer); + return errorCount; + } + + // Display control functions + const fontSizes = ['font-sm', 'font-md', 'font-lg', 'font-xl']; + const fontLabels = ['85%', '100%', '115%', '130%']; + + function getDisplayState(viewer) { return getLogViewerState(viewer); } + + function zoom(viewerId, delta) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + const state = getDisplayState(viewer); + const newIndex = state.currentFontIndex + delta; + if (newIndex < 0 || newIndex >= fontSizes.length) return; + + viewer.classList.remove(fontSizes[state.currentFontIndex]); + state.currentFontIndex = newIndex; + viewer.classList.add(fontSizes[state.currentFontIndex]); + updateZoomDisplay(viewerId); + } + + function updateZoomDisplay(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + const state = getDisplayState(viewer); + + // Derive zoom display ID from viewer ID + let displayId; + if (viewerId.startsWith('testLogViewer')) { + const index = viewerId.replace('testLogViewer', ''); + displayId = `testZoomDisplay${index}`; + } else { + displayId = 'zoomDisplay'; + } + + const display = document.getElementById(displayId); + if (display) { + display.textContent = fontLabels[state.currentFontIndex]; + } + } + + function toggleWrap(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + const state = getDisplayState(viewer); + + // Derive wrap button ID from viewer ID + let btnId; + if (viewerId.startsWith('testLogViewer')) { + const index = viewerId.replace('testLogViewer', ''); + btnId = `testWrapToggleBtn${index}`; + } else { + btnId = 'wrapToggleBtn'; + } + + const btn = document.getElementById(btnId); + if (!btn) return; + + state.wrapEnabled = !state.wrapEnabled; + + if (state.wrapEnabled) { + viewer.classList.remove('wrap-disabled'); + viewer.classList.add('wrap-enabled'); + btn.classList.add('active'); + btn.title = 'Line wrapping: ON (click to turn off)'; + // Update text for non-floating buttons + if (!btn.classList.contains('floating-wrap-btn')) { + btn.textContent = '↩ Wrap: On'; + } + } else { + viewer.classList.remove('wrap-enabled'); + viewer.classList.add('wrap-disabled'); + btn.classList.remove('active'); + btn.title = 'Line wrapping: OFF (click to turn on)'; + // Update text for non-floating buttons + if (!btn.classList.contains('floating-wrap-btn')) { + btn.textContent = '→ Wrap: Off'; + } + } + } + + function initDisplayControls(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + // Initialize state + const state = getDisplayState(viewer); + + // Set initial font size + viewer.classList.add(fontSizes[state.currentFontIndex]); + + // Set initial wrap state + if (state.wrapEnabled) { + viewer.classList.add('wrap-enabled'); + + // Set active class for wrap button + let btnId; + if (viewerId.startsWith('testLogViewer')) { + const index = viewerId.replace('testLogViewer', ''); + btnId = `testWrapToggleBtn${index}`; + } else { + btnId = 'wrapToggleBtn'; + } + const btn = document.getElementById(btnId); + if (btn) { + btn.classList.add('active'); + btn.title = 'Line wrapping: ON (click to turn off)'; + } + } + + // Add click handlers to line numbers + viewer.addEventListener('click', (e) => { + const lineNumber = e.target.closest('.log-line-number'); + if (lineNumber) { + const line = lineNumber.closest('.log-line'); + if (line) { + const lineContent = line.querySelector('.log-line-content'); + if (lineContent) { + const lineNum = Array.from(viewer.querySelectorAll('.log-line')).indexOf(line) + 1; + copyLineContent(lineNum, lineContent.textContent); + } + } + } + }); + + // Make line numbers clickable + viewer.querySelectorAll('.log-line-number').forEach(num => { + num.classList.add('log-line-number-clickable'); + }); + + // Update display + updateZoomDisplay(viewerId); + } + + function copyLineContent(lineNumber, lineContent) { + const textToCopy = `Line ${lineNumber}: ${lineContent}`; + + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(textToCopy).then(() => { + showCopyNotification('Line copied!'); + }).catch(err => { + console.error('Failed to copy:', err); + }); + } else { + // Fallback + const textarea = document.createElement('textarea'); + textarea.value = textToCopy; + textarea.style.position = 'fixed'; + textarea.style.opacity = '0'; + document.body.appendChild(textarea); + textarea.select(); + try { + document.execCommand('copy'); + showCopyNotification('Line copied!'); + } catch (err) { + console.error('Fallback copy failed:', err); + } + document.body.removeChild(textarea); + } + } + + function showNotification(message, type = 'success') { + const colors = { + success: '#10b981', + warning: '#f59e0b', + error: '#ef4444', + info: '#3b82f6' + }; + const notification = document.createElement('div'); + notification.textContent = message; + notification.style.cssText = `position:fixed;top:20px;right:20px;background:${colors[type] || colors.info};color:white;padding:10px 18px;border-radius:6px;font-size:13px;z-index:10000;box-shadow:0 4px 12px rgba(0,0,0,0.2);`; + document.body.appendChild(notification); + + setTimeout(() => { + notification.style.transition = 'opacity 0.3s ease'; + notification.style.opacity = '0'; + setTimeout(() => document.body.removeChild(notification), 300); + }, 2000); + } + + function showCopyNotification(message) { showNotification(message, 'success'); } + + // Search functionality + function getSearchState(viewer) { + const state = getLogViewerState(viewer); + // Map unified state fields to search-specific names for compatibility + return { + get currentIndex() { return state.searchIndex; }, + set currentIndex(v) { state.searchIndex = v; }, + get matches() { return state.searchMatches; }, + set matches(v) { state.searchMatches = v; }, + get originalContent() { return state.originalContent; }, + set originalContent(v) { state.originalContent = v; } + }; + } + + function performSearch(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + const state = getSearchState(viewer); + + // Get search input ID + let searchInputId, searchCounterId; + if (viewerId === 'runLogViewer') { + searchInputId = 'runLogSearchInput'; + searchCounterId = 'runLogSearchCounter'; + } else { + const index = viewerId.replace('testLogViewer', ''); + searchInputId = `testLogSearchInput${index}`; + searchCounterId = `testLogSearchCounter${index}`; + } + + const searchInput = document.getElementById(searchInputId); + const searchCounter = document.getElementById(searchCounterId); + + if (!searchInput || !searchCounter) return; + + const searchText = searchInput.value.trim(); + + // Save original content on first search + if (!state.originalContent) { + state.originalContent = viewer.innerHTML; + } + + // Clear previous search + viewer.innerHTML = state.originalContent; + state.matches = []; + state.currentIndex = -1; + + // Reset error navigation after DOM is rebuilt + const viewerState = getViewerState(viewer); + if (viewerState) { + viewerState.errorPositions = []; + viewerState.currentErrorIndex = -1; + } + + if (!searchText) { + searchCounter.textContent = '0/0'; + return; + } + + // Find and highlight all matches + const lines = viewer.querySelectorAll('.log-line'); + let matchCount = 0; + + lines.forEach(line => { + const contentDiv = line.querySelector('.log-line-content'); + if (!contentDiv) return; + + const text = contentDiv.textContent; + const lowerText = text.toLowerCase(); + const lowerSearch = searchText.toLowerCase(); + let index = 0; + let html = ''; + let lastIndex = 0; + + while ((index = lowerText.indexOf(lowerSearch, lastIndex)) !== -1) { + // Add text before match + html += escapeHtml(text.substring(lastIndex, index)); + // Add highlighted match + html += `${escapeHtml(text.substring(index, index + searchText.length))}`; + state.matches.push({ line, index: matchCount }); + matchCount++; + lastIndex = index + searchText.length; + } + + if (lastIndex > 0) { + html += escapeHtml(text.substring(lastIndex)); + contentDiv.innerHTML = html; + } + }); + + // Update counter + searchCounter.textContent = matchCount > 0 ? `1/${matchCount}` : '0/0'; + + // Highlight first match + if (matchCount > 0) { + state.currentIndex = 0; + highlightCurrentMatch(viewer, 0); + } + } + + function escapeHtml(text) { + const div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + } + + function highlightCurrentMatch(viewer, index) { + // Remove previous current highlight + const prevCurrent = viewer.querySelector('.search-highlight-current'); + if (prevCurrent) { + prevCurrent.classList.remove('search-highlight-current'); + prevCurrent.classList.add('search-highlight'); + } + + // Highlight new current match + const allHighlights = viewer.querySelectorAll('.search-highlight'); + if (index >= 0 && index < allHighlights.length) { + const currentHighlight = allHighlights[index]; + currentHighlight.classList.remove('search-highlight'); + currentHighlight.classList.add('search-highlight-current'); + + // Scroll to match + const line = currentHighlight.closest('.log-line'); + if (line) { + scrollToElement(line, viewer); + } + } + } + + function nextSearchResult(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + const state = getSearchState(viewer); + if (state.matches.length === 0) return; + + state.currentIndex = (state.currentIndex + 1) % state.matches.length; + highlightCurrentMatch(viewer, state.currentIndex); + updateSearchCounter(viewerId, state.currentIndex + 1, state.matches.length); + } + + function prevSearchResult(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + const state = getSearchState(viewer); + if (state.matches.length === 0) return; + + state.currentIndex = state.currentIndex - 1; + if (state.currentIndex < 0) { + state.currentIndex = state.matches.length - 1; + } + highlightCurrentMatch(viewer, state.currentIndex); + updateSearchCounter(viewerId, state.currentIndex + 1, state.matches.length); + } + + function updateSearchCounter(viewerId, current, total) { + let searchCounterId; + if (viewerId === 'runLogViewer') { + searchCounterId = 'runLogSearchCounter'; + } else { + const index = viewerId.replace('testLogViewer', ''); + searchCounterId = `testLogSearchCounter${index}`; + } + + const searchCounter = document.getElementById(searchCounterId); + if (searchCounter) { + searchCounter.textContent = `${current}/${total}`; + } + } + + // Actions menu functionality + function toggleActionsMenu(viewerId) { + const menuId = `actionsMenu_${viewerId}`; + const menu = document.getElementById(menuId); + if (!menu) return; + + // Close all other menus first + document.querySelectorAll('.log-actions-dropdown').forEach(m => { + if (m.id !== menuId) { + m.classList.remove('show'); + } + }); + + // Toggle current menu + menu.classList.toggle('show'); + } + + // Close menus when clicking outside + document.addEventListener('click', (e) => { + if (!e.target.closest('.log-actions-menu')) { + document.querySelectorAll('.log-actions-dropdown').forEach(m => { + m.classList.remove('show'); + }); + } + if (!e.target.closest('.log-filters-menu')) { + document.querySelectorAll('.log-filters-dropdown').forEach(m => { + m.classList.remove('show'); + }); + } + }); + + function copyAllLog(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + // Get all text content from log lines + const lines = viewer.querySelectorAll('.log-line-content'); + const text = Array.from(lines).map(line => line.textContent).join('\\n'); + + copyToClipboard(text, 'All log content copied!'); + toggleActionsMenu(viewerId); // Close menu after action + } + + function copyVisibleLog(viewerId) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + // Get visible text content (respecting any filters/search) + const text = viewer.textContent; + + copyToClipboard(text, 'Visible log content copied!'); + toggleActionsMenu(viewerId); + } + + function copyToClipboard(text, message) { + if (navigator.clipboard && navigator.clipboard.writeText) { + navigator.clipboard.writeText(text).then(() => { + showCopyNotification(message); + }).catch(err => { + console.error('Failed to copy:', err); + fallbackCopyToClipboard(text, message); + }); + } else { + fallbackCopyToClipboard(text, message); + } + } + + function fallbackCopyToClipboard(text, message) { + const textarea = document.createElement('textarea'); + textarea.value = text; + textarea.style.position = 'fixed'; + textarea.style.opacity = '0'; + document.body.appendChild(textarea); + textarea.select(); + try { + document.execCommand('copy'); + showCopyNotification(message); + } catch (err) { + console.error('Fallback copy failed:', err); + } + document.body.removeChild(textarea); + } + + function downloadLog(viewerId, extension, filename) { + const viewer = document.getElementById(viewerId); + if (!viewer) return; + + // Get all text content from log lines + const lines = viewer.querySelectorAll('.log-line-content'); + const text = Array.from(lines).map(line => line.textContent).join('\\n'); + + // Create blob and download + const blob = new Blob([text], { type: 'text/plain' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = `${filename}.${extension}`; + document.body.appendChild(a); + a.click(); + document.body.removeChild(a); + URL.revokeObjectURL(url); + + showCopyNotification(`Downloaded ${filename}.${extension}`); + toggleActionsMenu(viewerId); + } + + // Filters functionality + function getFilterState(viewer) { + return getLogViewerState(viewer).filters; + } + + function toggleFiltersMenu(viewerId) { + const menuId = `filtersMenu_${viewerId}`; + const menu = document.getElementById(menuId); + if (!menu) return; + + // Close all other menus + document.querySelectorAll('.log-filters-dropdown, .log-actions-dropdown').forEach(m => { + if (m.id !== menuId) { + m.classList.remove('show'); + } + }); + + // Toggle current menu + const wasOpen = menu.classList.contains('show'); + menu.classList.toggle('show'); + + // Count logs when opening + if (!wasOpen) { + const viewer = document.getElementById(viewerId); + if (viewer) { + countLogLevels(viewer, menuId); + } + } + } + + function countLogLevels(viewer, menuId) { + const menu = document.getElementById(menuId); + if (!menu) return; + + const counts = { + error: 0, + warning: 0, + info: 0, + debug: 0 + }; + + const lines = viewer.querySelectorAll('.log-line'); + lines.forEach(line => { + const text = line.textContent.toLowerCase(); + if (text.includes('error') || text.includes('failed') || text.includes('✗') || text.includes('❌')) { + counts.error++; + } else if (text.includes('warn')) { + counts.warning++; + } else if (text.includes('info') || text.includes('✓') || text.includes('✅')) { + counts.info++; + } else if (text.includes('debug')) { + counts.debug++; + } else { + counts.info++; // Default to info + } + }); + + // Update counts in UI + Object.keys(counts).forEach(level => { + const countSpan = menu.querySelector(`.filter-count[data-level="${level}"]`); + if (countSpan) { + countSpan.textContent = counts[level]; + } + }); + } + + function updateFilterState(viewerId) { + // This is called on checkbox change, but we apply filters on "Apply" click + // Just keep it for future real-time filtering if needed + } + + function applyFilters(viewerId) { + const viewer = document.getElementById(viewerId); + const menuId = `filtersMenu_${viewerId}`; + const menu = document.getElementById(menuId); + if (!viewer || !menu) return; + + // Get checked states + const checkboxes = menu.querySelectorAll('input[type="checkbox"]'); + const enabledLevels = { + error: false, + warning: false, + info: false, + debug: false + }; + + checkboxes.forEach(cb => { + const level = cb.getAttribute('data-level'); + if (level) { + enabledLevels[level] = cb.checked; + } + }); + + // Save state + const state = getFilterState(viewer); + Object.assign(state, enabledLevels); + + // Apply filters to log lines + const lines = viewer.querySelectorAll('.log-line'); + lines.forEach(line => { + const text = line.textContent.toLowerCase(); + let shouldShow = false; + + if (enabledLevels.error && (text.includes('error') || text.includes('failed') || text.includes('✗') || text.includes('❌'))) { + shouldShow = true; + } else if (enabledLevels.warning && text.includes('warn')) { + shouldShow = true; + } else if (enabledLevels.debug && text.includes('debug')) { + shouldShow = true; + } else if (enabledLevels.info) { + // Show everything else if info is enabled + shouldShow = true; + } + + line.style.display = shouldShow ? '' : 'none'; + }); + + // Close menu + toggleFiltersMenu(viewerId); + } + + function resetFilters(viewerId) { + const viewer = document.getElementById(viewerId); + const menuId = `filtersMenu_${viewerId}`; + const menu = document.getElementById(menuId); + if (!viewer || !menu) return; + + // Check all checkboxes + const checkboxes = menu.querySelectorAll('input[type="checkbox"]'); + checkboxes.forEach(cb => { + cb.checked = true; + }); + + // Show all lines + const lines = viewer.querySelectorAll('.log-line'); + lines.forEach(line => { + line.style.display = ''; + }); + + // Reset state + const state = getFilterState(viewer); + state.error = true; + state.warning = true; + state.info = true; + state.debug = true; + } + + function showRunDetails(runData) { + const modal = document.getElementById('testModal'); + const modalBody = document.getElementById('modalContent'); + + const passRate = runData.total_tests > 0 ? ((runData.passed_tests / runData.total_tests) * 100).toFixed(1) : '0.0'; + + let html = ` +
+ + +
+ +
+
+

Run Information

+
+
+
Run ID
+
#${runData.run_id}
+
+
+
Start Time
+
${formatTimestamp(runData.start_time)}
+
+
+
Total Runtime
+
${formatDuration(runData.total_runtime)}
+
+
+
Pass Rate
+
${passRate}%
+
+
+
+ +
+

Test Results

+
+
+
Total Tests
+
${runData.total_tests}
+
+
+
Passed
+
${runData.passed_tests}
+
+
+
Failed
+
${runData.failed_tests}
+
+
+
Skipped
+
${runData.total_tests - runData.passed_tests - runData.failed_tests}
+
+
+
+ +
+

Git Information

+
+
+
Branch
+
${runData.git_branch || 'unknown'}
+
+
+
Commit
+
${runData.git_commit || 'unknown'}
+
+
+
Working Directory
+
${runData.git_dirty ? '● Dirty' : '✓ Clean'}
+
+
+
+
+ +
+
+
+ Complete Test Run Output +
+
+ +
+
${svgIcon('copy', 14)} Copy All
+
${svgIcon('copy', 14)} Copy Visible
+
+
${svgIcon('download', 14)} Download .txt
+
${svgIcon('download', 14)} Download .log
+
+
+
+ +
+
Show:
+ + + + +
+ + +
+
+
+
+
+ + + + 0/0 +
+
+ + + + +
+
+
+
+ ${ansiToHtml(escapeHtml(runData.test_output_log || 'No logs available'))} +
+
+
+
+ + 100% + +
+
+ +
+
+
+
+
+ `; + + modalBody.innerHTML = html; + + // Apply syntax highlighting to run log viewer + const runLogViewer = document.getElementById('runLogViewer'); + if (runLogViewer) { + runLogViewer.innerHTML = highlightLogSyntax(runLogViewer.innerHTML); + runLogViewer.classList.add('with-line-numbers'); + // Initialize navigation and display controls after syntax highlighting + initLogNavigation(runLogViewer); + initDisplayControls('runLogViewer'); + } + + document.getElementById('testModalTitle').textContent = 'Run #' + runData.run_id + ' Details'; + modal.style.display = 'block'; + } + + function closeModal(id) { + if (!id) id = 'testModal'; + document.getElementById(id).style.display = 'none'; + } + + // ============================================================ + // SECTION: pprof Visualization + // ============================================================ + + function parsePprofHeap(content) { + const lines = content.split('\\n'); + const result = { totalSize: 0, totalObjects: 0, entries: [] }; + + // Parse header: heap profile: 12: 323920 [125: 1251160] @ heap/1048576 + const headerMatch = lines[0]?.match(/heap profile: (\\d+): (\\d+) \\[(\\d+): (\\d+)\\]/); + if (headerMatch) { + result.liveObjects = parseInt(headerMatch[1]); + result.liveSize = parseInt(headerMatch[2]); + result.totalObjects = parseInt(headerMatch[3]); + result.totalSize = parseInt(headerMatch[4]); + } + + // Parse entries + let currentEntry = null; + for (let i = 1; i < lines.length; i++) { + const line = lines[i]; + // Entry line: 1: 278528 [1: 278528] @ 0x... + const entryMatch = line.match(/^(\\d+): (\\d+) \\[(\\d+): (\\d+)\\] @/); + if (entryMatch) { + if (currentEntry) result.entries.push(currentEntry); + currentEntry = { + liveObjects: parseInt(entryMatch[1]), + liveSize: parseInt(entryMatch[2]), + totalObjects: parseInt(entryMatch[3]), + totalSize: parseInt(entryMatch[4]), + stack: [] + }; + } else if (line.startsWith('#') && currentEntry) { + // Stack frame: # 0x6dfa4b k8s.io/api/core/v1.(*Secret).Unmarshal+0x100b + const frameMatch = line.match(/#\\s+0x[0-9a-f]+\\s+(.+?)(?:\\s+\\/|$)/); + if (frameMatch) { + currentEntry.stack.push(frameMatch[1].trim()); + } + } + } + if (currentEntry) result.entries.push(currentEntry); + + // Sort by live size descending + result.entries.sort((a, b) => b.liveSize - a.liveSize); + return result; + } + + function parsePprofGoroutine(content) { + const lines = content.split('\\n'); + const result = { totalGoroutines: 0, stacks: [] }; + + // Parse header: goroutine profile: total 15 + const headerMatch = lines[0]?.match(/goroutine profile: total (\\d+)/); + if (headerMatch) { + result.totalGoroutines = parseInt(headerMatch[1]); + } + + // Parse entries + let currentStack = null; + for (let i = 1; i < lines.length; i++) { + const line = lines[i]; + // Entry line: 1 @ 0x... or count @ 0x... + const entryMatch = line.match(/^(\\d+) @/); + if (entryMatch) { + if (currentStack) result.stacks.push(currentStack); + currentStack = { + count: parseInt(entryMatch[1]), + frames: [] + }; + } else if (line.startsWith('#') && currentStack) { + const frameMatch = line.match(/#\\s+0x[0-9a-f]+\\s+(.+?)(?:\\s+\\/|$)/); + if (frameMatch) { + currentStack.frames.push(frameMatch[1].trim()); + } + } + } + if (currentStack) result.stacks.push(currentStack); + + // Sort by count descending + result.stacks.sort((a, b) => b.count - a.count); + return result; + } + + function formatBytes(bytes) { + if (bytes === 0) return '0 B'; + const k = 1024; + const sizes = ['B', 'KB', 'MB', 'GB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; + } + + function renderPprofHeap(parsed, containerId) { + const container = document.getElementById(containerId); + if (!container || !parsed) return; + + const topEntries = parsed.entries.slice(0, 8); + const maxSize = topEntries[0]?.liveSize || 1; + + let html = ` +
+
+ ${formatBytes(parsed.liveSize)} live + • + ${parsed.liveObjects?.toLocaleString() || 0} objects + • + ${formatBytes(parsed.totalSize)} total allocated +
+ +
+ How to interpret heap profile:
+ • Live memory — currently allocated and in use
+ • Top allocators — functions allocating most memory
+ • Look for unexpected large allocations or memory leaks

+ Deep analysis:
+ go tool pprof heap.pb.gz
+ Commands: top, web, list funcName +
+
+
+ `; + + topEntries.forEach((entry, idx) => { + const pct = (entry.liveSize / maxSize * 100).toFixed(0); + const funcName = entry.stack[0] || 'unknown'; + const shortName = funcName.split('.').pop() || funcName; + html += ` +
+
${escapeHtml(shortName)}
+
+
+ ${formatBytes(entry.liveSize)} +
+
+ `; + }); + + html += '
'; + container.innerHTML = html; + } + + function renderPprofGoroutine(parsed, containerId) { + const container = document.getElementById(containerId); + if (!container || !parsed) return; + + let html = ` +
+
+ ${parsed.totalGoroutines} goroutines + • + ${parsed.stacks.length} unique stacks +
+ +
+ How to interpret goroutine profile:
+ • Count (Nx) — number of goroutines with same stack
+ • High counts may indicate goroutine leaks
+ • Click stack to see full call trace

+ What to look for:
+ • Blocked goroutines (waiting on channels/mutexes)
+ • Unexpected goroutine accumulation over time
+ • Goroutines stuck in infinite loops +
+
+
+ `; + + parsed.stacks.slice(0, 12).forEach((stack, idx) => { + const topFrame = stack.frames[0] || 'unknown'; + const shortName = topFrame.split('.').pop() || topFrame; + html += ` +
+
+ ${stack.count}x + ${escapeHtml(shortName)} + ▶ +
+
+ ${stack.frames.map(f => `
${escapeHtml(f)}
`).join('')} +
+
+ `; + }); + + html += '
'; + container.innerHTML = html; + } + + function isPprofFile(filename) { + return filename.includes('pprof-heap') || filename.includes('pprof-goroutine'); + } + + function renderPprofFile(filename, content, containerId) { + if (filename.includes('pprof-heap')) { + renderPprofHeap(parsePprofHeap(content), containerId); + } else if (filename.includes('pprof-goroutine')) { + renderPprofGoroutine(parsePprofGoroutine(content), containerId); + } + } + + function copyAllTestNames() { + const rows = document.querySelectorAll('#resultsTable tbody tr'); + let names = []; + rows.forEach(row => { + if (row.style.display !== 'none') { + const data = JSON.parse(row.dataset.json); + names.push(data.test_name); + } + }); + + if (names.length > 0) { + navigator.clipboard.writeText(names.join('\\n')).then(() => { + const btn = document.querySelector('button[onclick="copyAllTestNames()"]'); + const originalText = btn.innerHTML; + btn.innerHTML = '✓ Copied ' + names.length + ' tests!'; + setTimeout(() => { + btn.innerHTML = originalText; + }, 2000); + }); + } else { + showNotification('No tests visible to copy', 'warning'); + } + } + + function ansiToHtml(text) { + if (!text) return ''; + + // Basic colors + const colors = { + 30: 'black', 31: '#ef4444', 32: '#10b981', 33: '#f59e0b', + 34: '#3b82f6', 35: '#d946ef', 36: '#06b6d4', 37: '#f8fafc', + 90: '#64748b', 91: '#f87171', 92: '#34d399', 93: '#fbbf24', + 94: '#60a5fa', 95: '#e879f9', 96: '#22d3ee', 97: '#ffffff' + }; + + // Backgrounds (40-47: normal, 100-107: bright) + const bgColors = { + 40: '#000000', 41: '#7f1d1d', 42: '#064e3b', 43: '#78350f', + 44: '#1e3a8a', 45: '#581c87', 46: '#164e63', 47: '#e5e7eb', + 100: '#374151', 101: '#991b1b', 102: '#065f46', 103: '#92400e', + 104: '#1e40af', 105: '#6b21a8', 106: '#0e7490', 107: '#f9fafb' + }; + + let html = ''; + let currentStyle = []; + + // Split by escape sequences + const parts = text.split(/(\\x1b\\[[0-9;]*m)/g); + + for (const part of parts) { + if (part.startsWith('\\x1b[')) { + const codes = part.slice(2, -1).split(';').map(Number); + for (const code of codes) { + if (code === 0) currentStyle = []; + else if (code === 1) currentStyle.push('font-weight:bold'); + else if (code === 2) currentStyle.push('opacity:0.7'); + else if (code === 4) currentStyle.push('text-decoration:underline'); + else if (colors[code]) currentStyle.push(`color:${colors[code]}`); + else if (bgColors[code]) currentStyle.push(`background-color:${bgColors[code]}`); + } + } else { + if (part) { + const style = currentStyle.length ? ` style="${currentStyle.join(';')}"` : ''; + html += `${escapeHtml(part)}`; + } + } + } + return html || text; + } + + // ============================================================ + // SECTION: Charts & Comparison + // ============================================================ + + function renderCharts() { + if (typeof Chart === 'undefined') return; + + const ctxRate = document.getElementById('passRateChart').getContext('2d'); + const ctxDuration = document.getElementById('durationChart').getContext('2d'); + + new Chart(ctxRate, { + type: 'line', + data: window.chartData.passRate, + options: { responsive: true, maintainAspectRatio: false } + }); + + new Chart(ctxDuration, { + type: 'line', + data: window.chartData.duration, + options: { responsive: true, maintainAspectRatio: false } + }); + } + + // Comparison + // Get run data by ID + function getRunData(runId) { + return window.chartData.runs.find(r => r.run_id == runId); + } + + // Update run info cards + function updateRunInfo() { + const runA = document.getElementById('runASelect').value; + const runB = document.getElementById('runBSelect').value; + + const runAData = getRunData(runA); + const runBData = getRunData(runB); + + if (runAData) { + const passRate = ((runAData.passed / runAData.total) * 100).toFixed(1); + document.getElementById('runAInfo').innerHTML = ` +
Run ID: ${runAData.run_id}
+
Date: ${new Date(runAData.timestamp).toLocaleString()}
+
Total Tests: ${runAData.total}
+
Passed: ${runAData.passed}
+
Failed: ${runAData.failed}
+
Pass Rate: ${passRate}%
+ `; + } + + if (runBData) { + const passRate = ((runBData.passed / runBData.total) * 100).toFixed(1); + document.getElementById('runBInfo').innerHTML = ` +
Run ID: ${runBData.run_id}
+
Date: ${new Date(runBData.timestamp).toLocaleString()}
+
Total Tests: ${runBData.total}
+
Passed: ${runBData.passed}
+
Failed: ${runBData.failed}
+
Pass Rate: ${passRate}%
+ `; + } + } + + function compareRuns() { + const runA = document.getElementById('runASelect').value; + const runB = document.getElementById('runBSelect').value; + + if (runA === runB) { + showNotification('Please select different runs to compare', 'warning'); + return; + } + + const newFailures = []; + const fixedTests = []; + const regressions = []; + + let totalRuntimeA = 0; + let totalRuntimeB = 0; + let testsCompared = 0; + + window.pivotData.forEach(row => { + const resA = row.runs[runA]; + const resB = row.runs[runB]; + + if (resA && resB) { + // Track runtime + if (resA.runtime) totalRuntimeA += resA.runtime; + if (resB.runtime) totalRuntimeB += resB.runtime; + testsCompared++; + + // New failures: A passed, B failed + if (resA.state === 'passed' && resB.state === 'failed') { + newFailures.push({...row, runtimeA: resA.runtime, runtimeB: resB.runtime}); + } + // Fixed tests: A failed, B passed + if (resA.state === 'failed' && resB.state === 'passed') { + fixedTests.push({...row, runtimeA: resA.runtime, runtimeB: resB.runtime}); + } + // Regressions: was flaky or occasionally failing, now consistently failing + if (row.is_flaky && resB.state === 'failed' && row.fail_count > 1) { + regressions.push({...row, runtimeA: resA.runtime, runtimeB: resB.runtime}); + } + } + }); + + // Update summary cards + document.getElementById('comparisonSummary').style.display = 'grid'; + document.getElementById('newFailuresCount').textContent = newFailures.length; + document.getElementById('fixedTestsCount').textContent = fixedTests.length; + document.getElementById('regressionsCount').textContent = regressions.length; + + // Calculate runtime diff + const runtimeDiff = totalRuntimeB - totalRuntimeA; + const runtimeDiffPercent = totalRuntimeA > 0 ? ((runtimeDiff / totalRuntimeA) * 100).toFixed(1) : 0; + const runtimeColor = runtimeDiff > 0 ? '#ef4444' : runtimeDiff < 0 ? '#10b981' : 'var(--text-primary)'; + const runtimeSign = runtimeDiff > 0 ? '+' : ''; + document.getElementById('runtimeDiff').innerHTML = `${runtimeSign}${runtimeDiff.toFixed(1)}s (${runtimeSign}${runtimeDiffPercent}%)`; + + // Render lists + document.getElementById('newFailuresList').innerHTML = newFailures.map(r => + `
+
${escapeHtml(r.test_name)}
+
+ Runtime: ${r.runtimeA ? r.runtimeA.toFixed(2) + 's' : 'N/A'} → ${r.runtimeB ? r.runtimeB.toFixed(2) + 's' : 'N/A'} +
+
` + ).join('') || '
No new failures
'; + + document.getElementById('fixedTestsList').innerHTML = fixedTests.map(r => + `
+
${escapeHtml(r.test_name)}
+
+ Runtime: ${r.runtimeA ? r.runtimeA.toFixed(2) + 's' : 'N/A'} → ${r.runtimeB ? r.runtimeB.toFixed(2) + 's' : 'N/A'} +
+
` + ).join('') || '
No fixed tests
'; + + document.getElementById('regressionsList').innerHTML = regressions.map(r => + `
+
${escapeHtml(r.test_name)}
+
+ Flakiness: ${r.flakiness_score ? r.flakiness_score.toFixed(0) + '% stable' : 'N/A'} | Fails: ${r.fail_count}/${r.total_runs} +
+
` + ).join('') || '
No regressions detected
'; + } + + // Initialize on load + document.addEventListener('DOMContentLoaded', function() { + updateRunInfo(); + }); + """ + +# --- Report Generator --- + +class ReportGenerator: + def __init__(self, results_dir: Path): + self.results_dir = results_dir + self.runs: List[TestRun] = [] + self.pivot_data: List[PivotRow] = [] + self.all_labels: Set[str] = set() + + def parse_results(self): + """Parse all test run results from the results directory.""" + if not self.results_dir.exists(): + return + + for run_dir in sorted(self.results_dir.glob("run-*")): + if not run_dir.is_dir(): + continue + + metadata_file = run_dir / "artifacts" / "metadata.json" + report_file = run_dir / "reports" / "report.json" + + if not metadata_file.exists() or not report_file.exists(): + continue + + try: + with open(metadata_file, 'r') as f: + metadata = json.load(f) + with open(report_file, 'r') as f: + report = json.load(f) + except Exception as e: + print(f"Error reading {run_dir}: {e}") + continue + + # Parse tests + tests = [] + spec_reports = report[0].get('SpecReports', []) if isinstance(report, list) else [] + + # Build artifact index once per run for O(1) lookup + artifact_index = self._build_artifact_index(run_dir) + + for spec in spec_reports: + if not spec.get('LeafNodeText'): + continue + + hierarchy = spec.get('ContainerHierarchyTexts', []) + leaf_text = spec['LeafNodeText'] + full_name = ' '.join(hierarchy + [leaf_text]) if hierarchy else leaf_text + + # Collect labels + labels = [] + for l_list in spec.get('ContainerHierarchyLabels', []): + if isinstance(l_list, list): labels.extend(l_list) + if spec.get('LeafNodeLabels'): labels.extend(spec.get('LeafNodeLabels')) + self.all_labels.update(labels) + + # Find artifacts using pre-built index + artifact_meta = self._find_artifacts(artifact_index, full_name) + + tests.append(TestResult( + name=full_name, + full_name=full_name, + leaf_text=leaf_text, + state=spec['State'], + runtime=spec['RunTime'] / 1e9, + failure_message=spec.get('FailureMessage', ''), + labels=labels, + container_hierarchy=hierarchy, + start_time=spec.get('StartTime', ''), + artifact_metadata=artifact_meta + )) + + # Calculate total runtime from tests if not in metadata + total_runtime = sum(t.runtime for t in tests) + + # Read test output log + test_output_log = "" + log_file = run_dir / "reports" / "test-output.log" + if log_file.exists(): + try: + with open(log_file, 'r', encoding='utf-8', errors='replace') as f: + lines = f.readlines() + if len(lines) > 10000: + test_output_log = f"Log truncated. Showing last 10,000 of {len(lines)} lines.\\n\\n" + "".join(lines[-10000:]) + else: + test_output_log = "".join(lines) + except Exception as e: + test_output_log = f"Error reading log file: {e}" + + self.runs.append(TestRun( + run_id=str(metadata.get('run_id', run_dir.name)), + start_time=metadata.get('start_time', datetime.now().isoformat()), + total_tests=metadata.get('total_tests', len(tests)), + passed_tests=metadata.get('passed_tests', len([t for t in tests if t.state == 'passed'])), + failed_tests=metadata.get('failed_tests', len([t for t in tests if t.state == 'failed'])), + environment=metadata.get('environment', {}), + total_runtime=total_runtime, + test_output_log=test_output_log, + tests=tests, + git_commit=metadata.get('git_commit', ''), + git_branch=metadata.get('git_branch', ''), + git_dirty=metadata.get('git_dirty', '') + )) + + # Sort runs by time (newest first) + self.runs.sort(key=lambda r: r.start_time, reverse=True) + self._build_pivot_data() + + def _build_artifact_index(self, run_dir: Path) -> Dict[str, Path]: + """Build name -> artifact_dir mapping for O(1) lookup.""" + index = {} + artifacts_dir = run_dir / "artifacts" + if not artifacts_dir.exists(): + return index + for artifact_dir in artifacts_dir.glob("*"): + if not artifact_dir.is_dir(): + continue + meta_file = artifact_dir / "metadata.json" + if meta_file.exists(): + try: + with open(meta_file) as f: + data = json.load(f) + if name := data.get('name'): + index[name.strip()] = artifact_dir + except: + pass + return index + + def _find_artifacts(self, artifact_index: Dict[str, Path], test_name: str) -> Optional[Dict[str, Any]]: + """Locate artifact metadata for a specific test using pre-built index.""" + artifact_dir = artifact_index.get(test_name.strip()) + if not artifact_dir: + return None + + meta_file = artifact_dir / "metadata.json" + if not meta_file.exists(): + return None + + try: + with open(meta_file) as f: + data = json.load(f) + + data['relative_path'] = str(artifact_dir.relative_to(self.results_dir)) + data['file_contents'] = {} + + # Read log files (last 500 lines) + for log in data.get('artifacts', {}).get('log_files', []): + lp = artifact_dir / log + if lp.exists(): + try: + with open(lp) as lf: + lines = lf.readlines() + content = ''.join(lines[-500:]) + data['file_contents'][log] = { + 'content': content, + 'type': 'log', + 'truncated': len(lines) > 500, + 'total_lines': len(lines) + } + except Exception as e: + data['file_contents'][log] = {'content': str(e), 'type': 'error'} + + # Read resource files (limit 50KB) + for res in data.get('artifacts', {}).get('resource_files', [])[:10]: + rp = artifact_dir / res + if rp.exists(): + try: + with open(rp) as rf: + content = rf.read() + if len(content) > 51200: content = content[:51200] + '\\n... (truncated)' + data['file_contents'][res] = {'content': content, 'type': 'resource'} + except Exception as e: + data['file_contents'][res] = {'content': str(e), 'type': 'error'} + + # Read event files + for evt in data.get('artifacts', {}).get('event_files', []): + ep = artifact_dir / evt + if ep.exists(): + try: + with open(ep) as ef: + data['file_contents'][evt] = {'content': ef.read(), 'type': 'events'} + except Exception as e: + data['file_contents'][evt] = {'content': str(e), 'type': 'error'} + + return data + except: + pass + return None + + def _build_pivot_data(self): + """Build the pivot table data structure and analyze flakiness.""" + all_names = sorted({t.full_name for run in self.runs for t in run.tests}) + + for name in all_names: + # Get container hierarchy from first occurrence of test + test_obj = None + for run in self.runs: + test_obj = next((t for t in run.tests if t.full_name == name), None) + if test_obj: + break + + row = PivotRow( + test_name=name, + full_test_name=name, + leaf_text=test_obj.leaf_text if test_obj else name.split(' ')[-1], + container_hierarchy=test_obj.container_hierarchy if test_obj else [] + ) + + results_sequence = [] + + for run in self.runs: + result = next((t for t in run.tests if t.full_name == name), None) + if result: + row.runs[run.run_id] = { + 'state': result.state, + 'runtime': result.runtime, + 'failure_message': result.failure_message, + 'labels': result.labels, + 'artifact_metadata': result.artifact_metadata + } + row.total_runs += 1 + row.total_runtime += result.runtime + row.min_runtime = min(row.min_runtime, result.runtime) + row.max_runtime = max(row.max_runtime, result.runtime) + + if result.state == 'passed': + row.pass_count += 1 + results_sequence.append('P') + elif result.state == 'failed': + row.fail_count += 1 + results_sequence.append('F') + else: + row.skip_count += 1 + results_sequence.append('S') + else: + row.runs[run.run_id] = None + + if row.total_runs > 0: + row.pass_rate = (row.pass_count / row.total_runs) * 100 + row.avg_runtime = row.total_runtime / row.total_runs + + # Flakiness Analysis + if row.total_runs >= 2 and row.pass_count > 0 and row.fail_count > 0: + row.is_flaky = True + row.flakiness_score = 100 - 2 * abs(row.pass_rate - 50) + + # Pattern detection + if len(results_sequence) >= 3: + is_alternating = True + for i in range(len(results_sequence) - 1): + if results_sequence[i] == results_sequence[i+1]: + is_alternating = False + break + if is_alternating: row.flakiness_pattern = 'alternating' + + self.pivot_data.append(row) + + # Sort by failure count + self.pivot_data.sort(key=lambda x: (x.fail_count, -x.pass_rate), reverse=True) + + def _generate_chart_data(self) -> str: + """Generate JSON data for Chart.js.""" + chronological_runs = sorted(self.runs, key=lambda r: r.start_time) + labels = [f"Run {r.run_id}" for r in chronological_runs] + + pass_rates = [] + durations = [] + runs_data = [] + + for r in chronological_runs: + rate = (r.passed_tests / r.total_tests * 100) if r.total_tests > 0 else 0 + pass_rates.append(round(rate, 1)) + durations.append(round(r.total_runtime, 1)) + + # Add run metadata for comparison + runs_data.append({ + 'run_id': r.run_id, + 'timestamp': r.start_time, + 'total': r.total_tests, + 'passed': r.passed_tests, + 'failed': r.failed_tests, + 'runtime': round(r.total_runtime, 1) + }) + + data = { + "passRate": { + "labels": labels, + "datasets": [{ + "label": "Pass Rate (%)", + "data": pass_rates, + "borderColor": "#10b981", + "backgroundColor": "rgba(16, 185, 129, 0.1)", + "fill": True + }] + }, + "duration": { + "labels": labels, + "datasets": [{ + "label": "Total Duration (s)", + "data": durations, + "borderColor": "#3b82f6", + "backgroundColor": "rgba(59, 130, 246, 0.1)", + "fill": True + }] + }, + "runs": runs_data + } + return json.dumps(data) + + def generate_html(self, output_file: Path): + """Generate the full HTML report.""" + + # Serialize pivot data for JS + pivot_json = json.dumps([ + { + 'test_name': r.test_name, + 'is_flaky': r.is_flaky, + 'pass_count': r.pass_count, + 'fail_count': r.fail_count, + 'total_runs': r.total_runs, + 'flakiness_score': r.flakiness_score, + 'runs': r.runs + } for r in self.pivot_data + ], default=str) + + html_content = f""" + + + + + E2E Test Report + + + + +
+
+
+

E2E Test Results

+

Generated on {datetime.now().strftime('%Y-%m-%d %H:%M')}

+
+
+ +
+
+ +
+ + + {f'' if len(self.runs) >= 2 else ''} +
+ + +
+
+
+
Total Runs
+
{len(self.runs)}
+
+
+
Total Tests
+
{len(self.pivot_data)}
+
+
+
Flaky Tests
+
{len([r for r in self.pivot_data if r.is_flaky])}
+ Tests that show inconsistent results across runs - sometimes passing, sometimes failing. These may indicate timing issues, race conditions, or environmental dependencies. +
+
+
Always Failing
+
{len([r for r in self.pivot_data if r.fail_count == r.total_runs and r.total_runs > 0])}
+ Tests that failed in every single run. These are consistently broken and require immediate attention. +
+
+
Avg Runtime
+
{format_duration(sum(r.total_runtime for r in self.runs) / len(self.runs)) if self.runs else 'N/A'}
+ Average total runtime across all test runs. Helps track performance trends over time. +
+
+
Pass Rate Trend
+
+ {self._get_pass_rate_trend()} +
+ Pass rate change compared to the previous run. ↑ indicates improvement, ↓ indicates more failures, → means stable. +
+ + +
+
Latest Run Summary
+
+
+
Pass Rate
+
{self._get_latest_pass_rate()}%
+
+
+
Failures
+
{self.runs[0].failed_tests if self.runs else 0}
+
+
+
Runtime
+
{format_duration(self.runs[0].total_runtime) if self.runs else 'N/A'}
+
+
+
+ + {self.runs[0].git_branch if self.runs and self.runs[0].git_branch else 'unknown'} + + {f'●' if self.runs and self.runs[0].git_dirty else ''} + + {svg_icon('copy', 12)} {self.runs[0].git_commit[:7] if self.runs and self.runs[0].git_commit else 'unknown'} + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+

{svg_icon('warning', 18, '#f59e0b')} Flaky Tests Detected

+
+ {''.join(f''' +
+
+ {r.flakiness_score:.0f}% Stable + Stability score based on pass/fail ratio. Lower values indicate more inconsistent behavior. + + {r.flakiness_pattern} + Pattern: {self._get_flakiness_pattern_description(r.flakiness_pattern)} + +
+
{r.test_name}
+
+ Pass: {r.pass_count} | Fail: {r.fail_count} +
+
+ ''' for r in self.pivot_data if r.is_flaky) or '

No flaky tests detected.

'} +
+
+
+ + +
+
+ + + + + +
+ +
+ + + + + + {''.join(f'' for r in self.runs)} + + + + {self._generate_table_rows()} + +
Test NameStatsRun {r.run_id}
{r.date_str}
{format_duration(r.total_runtime)}
+
+
+ + +
+
+ + + +
+ + +
+
+

Run A

+
+
+
+

Run B

+
+
+
+ + + + +
+
+

{svg_icon('error', 16, '#ef4444')} New Failures

+
+
+
+

{svg_icon('info', 16, '#10b981')} Fixed Tests

+
+
+
+

{svg_icon('warning', 16, '#f59e0b')} Regressions

+
+
+
+
+
+ + + + + + + +""" + with open(output_file, 'w') as f: + f.write(html_content) + print(f"Report generated at: {output_file}") + + def _get_latest_pass_rate(self) -> str: + if not self.runs: return "0.0" + latest = self.runs[0] + if latest.total_tests == 0: return "0.0" + return f"{(latest.passed_tests / latest.total_tests * 100):.1f}" + + def _get_pass_rate_trend(self) -> str: + """Get pass rate trend compared to previous run.""" + if len(self.runs) < 2: + return '—' + + latest = self.runs[0] + previous = self.runs[1] + + if latest.total_tests == 0 or previous.total_tests == 0: + return '—' + + latest_rate = (latest.passed_tests / latest.total_tests * 100) + previous_rate = (previous.passed_tests / previous.total_tests * 100) + diff = latest_rate - previous_rate + + if abs(diff) < 0.1: + return '→ Stable' + elif diff > 0: + return f'↑ +{diff:.1f}%' + else: + return f'↓ {diff:.1f}%' + + def _get_flakiness_pattern_description(self, pattern: str) -> str: + """Get description for flakiness pattern.""" + descriptions = { + 'intermittent': 'Fails randomly with no clear pattern', + 'occasional': 'Fails infrequently, mostly passes', + 'frequent': 'Fails often, passes sometimes', + 'alternating': 'Alternates between pass and fail', + 'unstable': 'Highly unpredictable behavior' + } + return descriptions.get(pattern, 'Unknown pattern') + + def _generate_hierarchical_name(self, container_hierarchy: List[str], leaf_text: str) -> str: + """Generate breadcrumb-style HTML representation of test name.""" + if not container_hierarchy: + # No hierarchy - just show the leaf text + return f'
{html.escape(leaf_text)}
' + + # Build breadcrumb structure + parts = [] + + # Add container hierarchy items + for i, container in enumerate(container_hierarchy): + level_class = f'level-{i}' if i == 0 else '' + parts.append(f'{html.escape(container)}') + parts.append('›') + + # Add leaf (actual test name) + parts.append(f'{html.escape(leaf_text)}') + + return f'
{"".join(parts)}
' + + def _generate_table_rows(self) -> str: + rows = [] + for row in self.pivot_data: + cells = [] + + # Test Name with hierarchy + hierarchical_name = self._generate_hierarchical_name(row.container_hierarchy, row.leaf_text) + # Full test name for copying (breadcrumb style) + full_test_name = ' › '.join(row.container_hierarchy + [row.leaf_text]) if row.container_hierarchy else row.leaf_text + cells.append(f''' +
+ {hierarchical_name} + +
+ ''') + + # Stats Cell + rate_class = 'rate-high' if row.pass_rate >= 90 else 'rate-medium' if row.pass_rate >= 50 else 'rate-low' + cells.append(f''' + +
+ {row.pass_rate:.1f}% + Pass rate: {row.pass_count} passed out of {row.total_runs} total runs +
+
{row.pass_count}✓ {row.fail_count}✗ / {row.total_runs}
+
avg: {row.avg_runtime:.1f}s
+ + ''') + + for run in self.runs: + result = row.runs.get(run.run_id) + if result: + # Runtime color + rt = result['runtime'] + rt_class = 'runtime-fast' if rt < 10 else 'runtime-medium' if rt < 30 else 'runtime-slow' + + # Serialize result for modal + data_json = json.dumps({ + 'test_name': row.test_name, + 'state': result['state'], + 'runtime': result['runtime'], + 'run_id': run.run_id, + 'failure_message': result['failure_message'], + 'artifact_metadata': result['artifact_metadata'] + }).replace("'", "'") + + cells.append(f''' + + {result["state"]} + {rt:.1f}s + + ''') + else: + cells.append('-') + + # Add metadata for filtering + row_meta = json.dumps({ + 'test_name': row.test_name, + 'is_flaky': row.is_flaky, + 'pass_count': row.pass_count, + 'fail_count': row.fail_count, + 'runs': row.runs + }, default=str).replace('"', '"') + + rows.append(f'{"".join(cells)}') + return "\n".join(rows) + +def main(): + if len(sys.argv) > 1: + results_dir = Path(sys.argv[1]) + else: + results_dir = Path.cwd() + + output_file = results_dir / "test_results_report.html" + + print(f"Scanning {results_dir}...") + generator = ReportGenerator(results_dir) + generator.parse_results() + + if not generator.runs: + print("No runs found.") + return + + generator.generate_html(output_file) + +if __name__ == "__main__": + main() diff --git a/test/e2e/testdata/normal-mode/agent.yaml b/test/e2e/testdata/normal-mode/agent.yaml new file mode 100644 index 00000000..f267c71e --- /dev/null +++ b/test/e2e/testdata/normal-mode/agent.yaml @@ -0,0 +1,7 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: Vector +metadata: + name: normal-agent +spec: + agent: + image: timberio/vector:0.40.0-alpine diff --git a/test/e2e/testdata/normal-mode/aggregator.yaml b/test/e2e/testdata/normal-mode/aggregator.yaml new file mode 100644 index 00000000..e12459f8 --- /dev/null +++ b/test/e2e/testdata/normal-mode/aggregator.yaml @@ -0,0 +1,11 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorAggregator +metadata: + name: normal-aggregator +spec: + image: timberio/vector:0.40.0-alpine + replicas: 1 + selector: + matchLabels: + app: test + role: aggregator diff --git a/test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns1.yaml b/test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns1.yaml new file mode 100644 index 00000000..73d0a0a6 --- /dev/null +++ b/test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns1.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cluster-monitored-pod-1 + labels: + app: cluster-test + cluster-monitor: enabled +spec: + containers: + - name: log-generator + image: busybox:1.36 + command: + - sh + - -c + - | + while true; do + echo '{"marker":"CLUSTER_MONITORED_NS1","message":"Cluster pipeline test from ns1"}' + sleep 5 + done + restartPolicy: Always diff --git a/test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns2.yaml b/test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns2.yaml new file mode 100644 index 00000000..ba81ae97 --- /dev/null +++ b/test/e2e/testdata/normal-mode/cluster-pipeline-pod-ns2.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cluster-monitored-pod-2 + namespace: test-normal-mode-isolated + labels: + app: cluster-test + cluster-monitor: enabled +spec: + containers: + - name: log-generator + image: busybox:1.36 + command: + - sh + - -c + - | + while true; do + echo '{"marker":"CLUSTER_MONITORED_NS2","message":"Cluster pipeline test from ns2"}' + sleep 5 + done + restartPolicy: Always diff --git a/test/e2e/testdata/normal-mode/cluster-pipeline.yaml b/test/e2e/testdata/normal-mode/cluster-pipeline.yaml new file mode 100644 index 00000000..68edad2e --- /dev/null +++ b/test/e2e/testdata/normal-mode/cluster-pipeline.yaml @@ -0,0 +1,24 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: ClusterVectorPipeline +metadata: + name: cluster-wide-pipeline +spec: + sources: + cluster_logs: + type: kubernetes_logs + extra_label_selector: "cluster-monitor=enabled" + transforms: + add_cluster_info: + type: remap + inputs: + - cluster_logs + source: | + .cluster_pipeline = "cluster-wide-pipeline" + .collected_at = now() + sinks: + console: + type: console + inputs: + - add_cluster_info + encoding: + codec: json diff --git a/test/e2e/testdata/normal-mode/namespace-isolation-ns.yaml b/test/e2e/testdata/normal-mode/namespace-isolation-ns.yaml new file mode 100644 index 00000000..03a385a5 --- /dev/null +++ b/test/e2e/testdata/normal-mode/namespace-isolation-ns.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-normal-mode-isolated diff --git a/test/e2e/testdata/normal-mode/namespace-isolation-pipeline.yaml b/test/e2e/testdata/normal-mode/namespace-isolation-pipeline.yaml new file mode 100644 index 00000000..4e29ab7e --- /dev/null +++ b/test/e2e/testdata/normal-mode/namespace-isolation-pipeline.yaml @@ -0,0 +1,17 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorPipeline +metadata: + name: isolated-pipeline + namespace: test-normal-mode-isolated +spec: + sources: + my_namespace_logs: + type: kubernetes_logs + # Should only see logs from test-normal-mode-isolated namespace + sinks: + console: + type: console + inputs: + - my_namespace_logs + encoding: + codec: json diff --git a/test/e2e/testdata/normal-mode/namespace-isolation-pod-isolated.yaml b/test/e2e/testdata/normal-mode/namespace-isolation-pod-isolated.yaml new file mode 100644 index 00000000..fd28d45b --- /dev/null +++ b/test/e2e/testdata/normal-mode/namespace-isolation-pod-isolated.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: isolated-pod + namespace: test-normal-mode-isolated + labels: + app: isolated-app +spec: + containers: + - name: log-generator + image: busybox:1.36 + command: + - sh + - -c + - | + while true; do + echo '{"marker":"ISOLATED_NAMESPACE","message":"This is from isolated namespace"}' + sleep 5 + done + restartPolicy: Always diff --git a/test/e2e/testdata/normal-mode/namespace-isolation-pod-main.yaml b/test/e2e/testdata/normal-mode/namespace-isolation-pod-main.yaml new file mode 100644 index 00000000..ba5b6288 --- /dev/null +++ b/test/e2e/testdata/normal-mode/namespace-isolation-pod-main.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: main-namespace-pod + labels: + app: main-app +spec: + containers: + - name: log-generator + image: busybox:1.36 + command: + - sh + - -c + - | + while true; do + echo '{"marker":"MAIN_NAMESPACE","message":"This is from main namespace"}' + sleep 5 + done + restartPolicy: Always diff --git a/test/e2e/testdata/normal-mode/pipeline-aggregator-role.yaml b/test/e2e/testdata/normal-mode/pipeline-aggregator-role.yaml new file mode 100644 index 00000000..568fc21d --- /dev/null +++ b/test/e2e/testdata/normal-mode/pipeline-aggregator-role.yaml @@ -0,0 +1,28 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorPipeline +metadata: + name: aggregator-pipeline + labels: + app: test + role: aggregator + annotations: + vector.kaasops.io/role: "Aggregator" +spec: + sources: + vector_source: + type: vector + address: "0.0.0.0:9000" + transforms: + process: + type: remap + inputs: + - vector_source + source: | + .processed = true + sinks: + console: + type: console + inputs: + - process + encoding: + codec: json diff --git a/test/e2e/testdata/normal-mode/pipeline-basic.yaml b/test/e2e/testdata/normal-mode/pipeline-basic.yaml new file mode 100644 index 00000000..9f53aefc --- /dev/null +++ b/test/e2e/testdata/normal-mode/pipeline-basic.yaml @@ -0,0 +1,16 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorPipeline +metadata: + name: basic-pipeline +spec: + sources: + kubernetes_logs: + type: kubernetes_logs + extra_label_selector: "app=test-app" + sinks: + console: + type: console + inputs: + - kubernetes_logs + encoding: + codec: json diff --git a/test/e2e/testdata/normal-mode/pipeline-complex.yaml b/test/e2e/testdata/normal-mode/pipeline-complex.yaml new file mode 100644 index 00000000..177be4c1 --- /dev/null +++ b/test/e2e/testdata/normal-mode/pipeline-complex.yaml @@ -0,0 +1,34 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorPipeline +metadata: + name: complex-pipeline +spec: + sources: + kubernetes_logs: + type: kubernetes_logs + extra_label_selector: "app=test-app" + transforms: + parse: + type: remap + inputs: + - kubernetes_logs + source: | + .parsed = parse_json!(.message) + filter: + type: filter + inputs: + - parse + condition: '.level == "info"' + sinks: + console_all: + type: console + inputs: + - parse + encoding: + codec: json + console_filtered: + type: console + inputs: + - filter + encoding: + codec: text diff --git a/test/e2e/testdata/normal-mode/pipeline-deletable.yaml b/test/e2e/testdata/normal-mode/pipeline-deletable.yaml new file mode 100644 index 00000000..c91f2cd8 --- /dev/null +++ b/test/e2e/testdata/normal-mode/pipeline-deletable.yaml @@ -0,0 +1,16 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorPipeline +metadata: + name: deletable-pipeline +spec: + sources: + kubernetes_logs: + type: kubernetes_logs + extra_label_selector: "app=test-app" + sinks: + console: + type: console + inputs: + - kubernetes_logs + encoding: + codec: json diff --git a/test/e2e/testdata/normal-mode/pipeline-kubernetes-logs.yaml b/test/e2e/testdata/normal-mode/pipeline-kubernetes-logs.yaml new file mode 100644 index 00000000..3cb0f0ac --- /dev/null +++ b/test/e2e/testdata/normal-mode/pipeline-kubernetes-logs.yaml @@ -0,0 +1,24 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorPipeline +metadata: + name: k8s-logs-pipeline +spec: + sources: + kubernetes_logs: + type: kubernetes_logs + extra_label_selector: "app=test-app" + transforms: + filter: + type: filter + inputs: + - kubernetes_logs + condition: + type: vrl + source: '.level != "debug"' + sinks: + console: + type: console + inputs: + - filter + encoding: + codec: json diff --git a/test/e2e/testdata/normal-mode/pipeline-template.yaml b/test/e2e/testdata/normal-mode/pipeline-template.yaml new file mode 100644 index 00000000..1a2ef5e6 --- /dev/null +++ b/test/e2e/testdata/normal-mode/pipeline-template.yaml @@ -0,0 +1,16 @@ +apiVersion: observability.kaasops.io/v1alpha1 +kind: VectorPipeline +metadata: + name: {{INDEX}} +spec: + sources: + kubernetes_logs: + type: kubernetes_logs + extra_label_selector: "app=test-app" + sinks: + console: + type: console + inputs: + - kubernetes_logs + encoding: + codec: json diff --git a/test/e2e/testdata/normal-mode/test-app-pod.yaml b/test/e2e/testdata/normal-mode/test-app-pod.yaml new file mode 100644 index 00000000..6eb8d1e9 --- /dev/null +++ b/test/e2e/testdata/normal-mode/test-app-pod.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-app + labels: + app: test-app +spec: + containers: + - name: log-generator + image: busybox:1.36 + command: + - sh + - -c + - | + while true; do + echo '{"level":"info","message":"Test log from test-app","timestamp":"'$(date -Iseconds)'"}' + echo '{"level":"debug","message":"Debug log should be filtered","timestamp":"'$(date -Iseconds)'"}' + sleep 5 + done + restartPolicy: Always diff --git a/test/e2e/testdata_helper.go b/test/e2e/testdata_helper.go new file mode 100644 index 00000000..16b70346 --- /dev/null +++ b/test/e2e/testdata_helper.go @@ -0,0 +1,17 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e