diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 2a64dc05..940ba139 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -2,13 +2,6 @@ name: formae-e2e-tests on: workflow_dispatch: - inputs: - plugin_refs: - description: >- - Plugin ref overrides as comma-separated name=ref pairs - (e.g., aws=feat/msgpack,azure=feat/msgpack). Leave empty for main. - required: false - default: '' pull_request: branches: [ "main" ] @@ -85,23 +78,45 @@ jobs: strategy: fail-fast: false matrix: - test: - - TestReconcileApply - - TestPatchApply - - TestDiscovery - - TestSoftReconcile - - TestHardReconcile - - TestReplace - - TestAutoReconcile - - TestTTL - - TestCancelCommand - - TestCascadeDelete - - TestTargetResolvable - - TestProjectInit - - TestSimulateApply - - TestExtractAndReapply - - TestAuthBasic - - TestPluginConfig + include: + - test: TestReconcileApply + plugins: aws azure + - test: TestPatchApply + plugins: aws azure + - test: TestDiscovery + plugins: aws azure + - test: TestSoftReconcile + plugins: aws azure + - test: TestHardReconcile + plugins: aws azure + - test: TestReplace + plugins: aws azure + - test: TestAutoReconcile + plugins: aws + - test: TestTTL + plugins: aws + - test: TestCancelCommand + plugins: aws + - test: TestCascadeDelete + plugins: aws + - test: TestTargetResolvable + plugins: compose grafana + - test: TestProjectInit + plugins: aws azure + - test: TestSimulateApply + plugins: aws + - test: TestExtractAndReapply + plugins: aws + - test: TestEvalDefault + plugins: aws + - test: TestEvalSchemaLocationLocal + plugins: aws + - test: TestExtractSchemaLocationLocal + plugins: aws + - test: TestAuthBasic + plugins: auth-basic + - test: TestPluginConfig + plugins: sftp steps: - uses: actions/checkout@v4 @@ -155,33 +170,84 @@ jobs: tenant-id: ${{ secrets.AZURE_TENANT_ID }} subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - name: Build plugin repo list with ref overrides + - name: Install plugins via orbital run: | - # Start with default repos from Makefile - REPOS="$(grep -A 20 '^EXTERNAL_PLUGIN_REPOS' Makefile | grep '\.git' | sed 's/\\$//' | xargs)" - - # Apply ref overrides from input (e.g., "aws=feat/msgpack,azure=feat/msgpack") - REFS="${{ github.event.inputs.plugin_refs }}" - if [ -n "$REFS" ]; then - IFS=',' read -ra PAIRS <<< "$REFS" - for pair in "${PAIRS[@]}"; do - name="${pair%%=*}" - ref="${pair##*=}" - REPOS=$(echo "$REPOS" | sed "s|formae-plugin-${name}.git\(@[^ ]*\)\{0,1\}|formae-plugin-${name}.git@${ref}|g") - echo "Pinning ${name} to ref: ${ref}" - done + set -euo pipefail + + # Build the formae binary and stage it in an installer-shaped tree + # so orbital (in Embedded mode, derives treeRoot from binary path) + # uses dist/e2e/ as the install root. The bootstrap agent below + # installs plugins through the same orbital tree the test agents + # will later read from (via the SystemPluginDir scan added in + # the multi-source plugin discovery refactor). + make build + mkdir -p dist/e2e/bin dist/e2e/.ops + cp formae dist/e2e/bin/formae + + # Minimal bootstrap config — uses the default artifacts.repositories + # (pel#stable + community#stable), no auth, no discovery, no sync. + cat > /tmp/bootstrap.conf.pkl <<'EOF' + amends "formae:/Config.pkl" + + agent { + server { port = 49684 } + datastore { + datastoreType = "sqlite" + sqlite { filePath = "/tmp/bootstrap.db" } + } + synchronization { enabled = false } + discovery { enabled = false } + logging { consoleLogLevel = "info" } + } + + cli { + api { port = 49684 } + disableUsageReporting = true + } + EOF + + # Start the bootstrap agent in the background. + FORMAE_PID_FILE=/tmp/bootstrap.pid \ + dist/e2e/bin/formae agent start --config /tmp/bootstrap.conf.pkl \ + > /tmp/bootstrap-agent.log 2>&1 & + AGENT_PID=$! + + # Wait up to 30s for the agent to come up. + for i in $(seq 1 30); do + if curl -sf http://localhost:49684/api/v1/health >/dev/null 2>&1; then + echo "Bootstrap agent ready after ${i}s" + break + fi + sleep 1 + done + if ! curl -sf http://localhost:49684/api/v1/health >/dev/null 2>&1; then + echo "::error::Bootstrap agent did not become healthy" + cat /tmp/bootstrap-agent.log || true + exit 1 fi - echo "EXTERNAL_PLUGIN_REPOS=${REPOS}" >> "$GITHUB_ENV" + # Install only the plugins this test needs. The agent refreshes + # orbital's repository cache at startup, so the install candidate + # solver has fresh metadata to work with. + dist/e2e/bin/formae plugin install ${{ matrix.plugins }} --channel stable + + # Stop the bootstrap agent so the test harness can spawn its own. + kill -TERM "$AGENT_PID" || true + wait "$AGENT_PID" 2>/dev/null || true + rm -f /tmp/bootstrap.pid /tmp/bootstrap.db /tmp/bootstrap.conf.pkl - name: Running ${{ matrix.test }} env: AWS_PROFILE: e2e-test AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + # Point setup_pkl.sh at the orbital-installed plugins. The agent's + # own multi-source discovery already finds them via the binary's + # SystemPluginDir, but setup_pkl.sh is a shell script that needs + # an explicit dir to read PklProject manifests from. + FORMAE_PLUGIN_DIR: ${{ github.workspace }}/dist/e2e/formae/plugins run: >- make test-e2e E2E_RUN_FLAGS="-run ${{ matrix.test }}" - EXTERNAL_PLUGIN_REPOS="${EXTERNAL_PLUGIN_REPOS}" cleanup: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 974b5d59..1c6b5212 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -139,12 +139,6 @@ jobs: - name: Publish core PKL schema to S3 run: make publish-pkl - - name: Package external plugin PKL schemas - run: make pkg-external-pkl - - - name: Publish external plugin PKL schemas to S3 - run: make publish-external-pkl - pkg_opkg: uses: "./.github/workflows/package.yml" secrets: inherit diff --git a/Dockerfile b/Dockerfile index 59a03235..68cc18b1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,9 +8,14 @@ RUN if [ -z "$VERSION" ]; then echo "VERSION is required"; exit 1; fi ENV PATH=/opt/pel/bin:$PATH RUN useradd -m -s /bin/bash pel + +# Install formae + the standard metapackage. The metapackage's `requires` +# resolve at install time and pull in the curated default plugin set +# (aws, azure, gcp, oci, ovh, auth-basic) — replacing the legacy bundled- +# plugins-from-binary extraction step. RUN apt-get update && \ apt-get install -y jq curl && \ - HOME=/home/pel /bin/bash -e -c "$(curl -fsSL https://hub.platform.engineering/get/setup.sh)" -- install --yes --channel ${CHANNEL} formae@${VERSION} && \ + HOME=/home/pel /bin/bash -e -c "$(curl -fsSL https://hub.platform.engineering/get/setup.sh)" -- install --yes --channel ${CHANNEL} formae@${VERSION} standard && \ apt-get remove -y jq curl && \ apt-get autoremove -y --purge && \ apt-get clean && \ diff --git a/Makefile b/Makefile index bdf57917..1a5d9611 100644 --- a/Makefile +++ b/Makefile @@ -24,29 +24,11 @@ VERSION := $(shell echo "$(RAW_VERSION)" | cut -d'-' -f1) # suffix. CHANNEL := $(or $(word 2,$(subst -, ,$(RAW_VERSION))),stable) -# External plugin Git repositories to bundle. -# Append @branch or @tag to pin a specific ref (e.g., ...aws.git@feat/msgpack). -# Without @ref, the default branch (main) is used. -EXTERNAL_PLUGIN_REPOS ?= \ - https://github.com/platform-engineering-labs/formae-plugin-auth-basic.git \ - https://github.com/platform-engineering-labs/formae-plugin-aws.git@feat/0.1.6-attachesto \ - https://github.com/platform-engineering-labs/formae-plugin-azure.git \ - https://github.com/platform-engineering-labs/formae-plugin-compose.git \ - https://github.com/platform-engineering-labs/formae-plugin-gcp.git \ - https://github.com/platform-engineering-labs/formae-plugin-grafana.git \ - https://github.com/platform-engineering-labs/formae-plugin-oci.git \ - https://github.com/platform-engineering-labs/formae-plugin-ovh.git \ - https://github.com/platform-engineering-labs/formae-plugin-sftp.git - -# Directory for cloned plugins -PLUGINS_CACHE := .plugins - clean: rm -rf .out/ rm -rf dist/ rm -rf formae rm -rf version.semver - rm -rf $(PLUGINS_CACHE) clean-pel: rm -rf ~/.pel/* @@ -58,141 +40,13 @@ build: install-gremlins: go install github.com/go-gremlins/gremlins/cmd/gremlins@latest -## fetch-external-plugins: Clone/update external plugin repositories -## Supports @ref suffix on repo URLs (e.g., repo.git@feat/branch) -fetch-external-plugins: - @mkdir -p $(PLUGINS_CACHE) - @for entry in $(EXTERNAL_PLUGIN_REPOS); do \ - ref=$$(echo "$$entry" | grep -o '@[^@]*$$' | sed 's/^@//'); \ - repo=$$(echo "$$entry" | sed 's/@[^@]*$$//'); \ - name=$$(basename $$repo .git); \ - if [ -d "$(PLUGINS_CACHE)/$$name" ]; then \ - echo "Updating $$name..."; \ - git -C "$(PLUGINS_CACHE)/$$name" fetch origin; \ - if [ -n "$$ref" ]; then \ - echo "Checking out $$name ref: $$ref"; \ - git -C "$(PLUGINS_CACHE)/$$name" checkout "origin/$$ref" --detach 2>/dev/null \ - || git -C "$(PLUGINS_CACHE)/$$name" checkout "$$ref" --detach; \ - else \ - git -C "$(PLUGINS_CACHE)/$$name" checkout origin/HEAD --detach 2>/dev/null \ - || git -C "$(PLUGINS_CACHE)/$$name" pull --ff-only; \ - fi; \ - else \ - echo "Cloning $$name..."; \ - if [ -n "$$ref" ]; then \ - git clone --depth 1 --branch "$$ref" $$repo "$(PLUGINS_CACHE)/$$name"; \ - else \ - git clone --depth 1 $$repo "$(PLUGINS_CACHE)/$$name"; \ - fi; \ - . ./scripts/ci/track-event.sh && formae_track_event "ci_repo_clone" "cloned_repo=$$name"; \ - fi; \ - done - -## build-external-plugins: Build all external plugins -build-external-plugins: fetch-external-plugins - @for entry in $(EXTERNAL_PLUGIN_REPOS); do \ - repo=$$(echo "$$entry" | sed 's/@[^@]*$$//'); \ - name=$$(basename $$repo .git); \ - echo "Building $$name..."; \ - cd "$(PLUGINS_CACHE)/$$name" && \ - if grep -q 'formae/pkg/auth' go.mod 2>/dev/null; then \ - go mod edit -replace github.com/platform-engineering-labs/formae/pkg/auth=$(CURDIR)/pkg/auth; \ - fi && \ - if grep -q 'formae/pkg/model' go.mod 2>/dev/null; then \ - go mod edit -replace github.com/platform-engineering-labs/formae/pkg/model=$(CURDIR)/pkg/model; \ - fi && \ - if grep -q 'formae/pkg/plugin' go.mod 2>/dev/null; then \ - go mod edit -replace github.com/platform-engineering-labs/formae/pkg/plugin=$(CURDIR)/pkg/plugin; \ - fi && \ - go mod tidy && \ - cd $(CURDIR); \ - $(MAKE) -C "$(PLUGINS_CACHE)/$$name" build; \ - done - -## install-external-plugins: Install external plugins to user directory (wipes existing versions) -install-external-plugins: build-external-plugins - @for entry in $(EXTERNAL_PLUGIN_REPOS); do \ - repo=$$(echo "$$entry" | sed 's/@[^@]*$$//'); \ - name=$$(basename $$repo .git); \ - plugin_dir="$(PLUGINS_CACHE)/$$name"; \ - plugin_type=$$(pkl eval -x 'if (this.hasProperty("type")) type else "resource"' "$$plugin_dir/formae-plugin.pkl" 2>/dev/null || echo "resource"); \ - version=$$(pkl eval -x 'version' "$$plugin_dir/formae-plugin.pkl"); \ - plugin_name=$$(pkl eval -x 'name' "$$plugin_dir/formae-plugin.pkl"); \ - if [ "$$plugin_type" = "auth" ]; then \ - dest="$$HOME/.pel/formae/plugins/$$plugin_name/v$$version"; \ - echo "Installing auth plugin: $$plugin_name v$$version to $$dest"; \ - rm -rf "$$HOME/.pel/formae/plugins/$$plugin_name"; \ - mkdir -p "$$dest"; \ - cp "$$plugin_dir/bin/$$plugin_name" "$$dest/$$plugin_name"; \ - cp "$$plugin_dir/formae-plugin.pkl" "$$dest/"; \ - if [ -d "$$plugin_dir/schema/pkl" ]; then \ - mkdir -p "$$dest/schema"; \ - cp -r "$$plugin_dir/schema/pkl" "$$dest/schema/"; \ - fi; \ - if [ -f "$$plugin_dir/schema/Config.pkl" ]; then \ - mkdir -p "$$dest/schema"; \ - cp "$$plugin_dir/schema/Config.pkl" "$$dest/schema/"; \ - fi; \ - else \ - namespace=$$(pkl eval -x 'namespace' "$$plugin_dir/formae-plugin.pkl" | tr '[:upper:]' '[:lower:]'); \ - dest="$$HOME/.pel/formae/plugins/$$namespace/v$$version"; \ - echo "Installing resource plugin: $$namespace v$$version to $$dest"; \ - rm -rf "$$HOME/.pel/formae/plugins/$$namespace"; \ - mkdir -p "$$dest/schema"; \ - cp "$$plugin_dir/bin/$$plugin_name" "$$dest/$$namespace"; \ - cp "$$plugin_dir/formae-plugin.pkl" "$$dest/"; \ - cp -r "$$plugin_dir/schema/pkl" "$$dest/schema/"; \ - if [ -f "$$plugin_dir/schema/Config.pkl" ]; then \ - cp "$$plugin_dir/schema/Config.pkl" "$$dest/schema/"; \ - fi; \ - fi; \ - done - @echo "External plugins installed successfully." - build-debug: go build ${DEBUG_GOFLAGS} -o formae cmd/formae/main.go -pkg-bin: clean build build-external-plugins +pkg-bin: clean build echo '${VERSION}' > ./version.semver mkdir -p ./dist/pel/bin cp -Rp ./formae ./dist/pel/bin - # Package external plugins (resource + auth) - @for entry in $(EXTERNAL_PLUGIN_REPOS); do \ - repo=$$(echo "$$entry" | sed 's/@[^@]*$$//'); \ - name=$$(basename $$repo .git); \ - plugin_dir="$(PLUGINS_CACHE)/$$name"; \ - plugin_type=$$(pkl eval -x 'if (this.hasProperty("type")) type else "resource"' "$$plugin_dir/formae-plugin.pkl" 2>/dev/null || echo "resource"); \ - version=$$(pkl eval -x 'version' "$$plugin_dir/formae-plugin.pkl"); \ - plugin_name=$$(pkl eval -x 'name' "$$plugin_dir/formae-plugin.pkl"); \ - if [ "$$plugin_type" = "auth" ]; then \ - dest="./dist/pel/formae/plugins/$$plugin_name/v$$version"; \ - echo "Packaging auth plugin: $$plugin_name v$$version"; \ - mkdir -p "$$dest"; \ - cp "$$plugin_dir/bin/$$plugin_name" "$$dest/$$plugin_name"; \ - cp "$$plugin_dir/formae-plugin.pkl" "$$dest/"; \ - if [ -d "$$plugin_dir/schema/pkl" ]; then \ - mkdir -p "$$dest/schema"; \ - cp -r "$$plugin_dir/schema/pkl" "$$dest/schema/"; \ - fi; \ - if [ -f "$$plugin_dir/schema/Config.pkl" ]; then \ - mkdir -p "$$dest/schema"; \ - cp "$$plugin_dir/schema/Config.pkl" "$$dest/schema/"; \ - fi; \ - else \ - namespace=$$(pkl eval -x 'namespace' "$$plugin_dir/formae-plugin.pkl" | tr '[:upper:]' '[:lower:]'); \ - dest="./dist/pel/formae/plugins/$$namespace/v$$version"; \ - echo "Packaging resource plugin: $$namespace v$$version"; \ - mkdir -p "$$dest/schema"; \ - cp "$$plugin_dir/bin/$$plugin_name" "$$dest/$$namespace"; \ - cp "$$plugin_dir/formae-plugin.pkl" "$$dest/"; \ - cp -r "$$plugin_dir/schema/pkl" "$$dest/schema/"; \ - if [ -f "$$plugin_dir/schema/Config.pkl" ]; then \ - cp "$$plugin_dir/schema/Config.pkl" "$$dest/schema/"; \ - fi; \ - mkdir -p "./dist/pel/formae/examples/$$plugin_name"; \ - cp -r "$$plugin_dir/examples/"* "./dist/pel/formae/examples/$$plugin_name/" 2>/dev/null || true; \ - fi; \ - done gen-pkl: echo '${VERSION}' > ./version.semver @@ -209,49 +63,6 @@ pkg-pkl: publish-pkl: aws s3 sync .out/formae@${VERSION} s3://hub.platform.engineering/plugins/pkl/schema/pkl/formae/ -## gen-external-pkl: Resolve external plugin PKL schemas (requires formae to be published first) -gen-external-pkl: fetch-external-plugins - @for entry in $(EXTERNAL_PLUGIN_REPOS); do \ - repo=$$(echo "$$entry" | sed 's/@[^@]*$$//'); \ - name=$$(basename $$repo .git); \ - plugin_dir="$(PLUGINS_CACHE)/$$name"; \ - schema_dir="$$plugin_dir/schema/pkl"; \ - if [ -d "$$schema_dir" ] && [ -f "$$schema_dir/PklProject" ]; then \ - version=$$(pkl eval -x 'version' "$$plugin_dir/formae-plugin.pkl"); \ - echo "$$version" > "$$schema_dir/VERSION"; \ - echo "Resolving PKL schema for $$name (v$$version)..."; \ - pkl project resolve "$$schema_dir"; \ - fi \ - done - -## pkg-external-pkl: Package external plugin PKL schemas -pkg-external-pkl: gen-external-pkl - @for entry in $(EXTERNAL_PLUGIN_REPOS); do \ - repo=$$(echo "$$entry" | sed 's/@[^@]*$$//'); \ - name=$$(basename $$repo .git); \ - schema_dir="$(PLUGINS_CACHE)/$$name/schema/pkl"; \ - if [ -d "$$schema_dir" ] && [ -f "$$schema_dir/PklProject" ]; then \ - echo "Packaging PKL schema for $$name..."; \ - pkl project package "$$schema_dir" --skip-publish-check; \ - fi \ - done - -## publish-external-pkl: Publish external plugin PKL schemas to S3 -publish-external-pkl: - @for entry in $(EXTERNAL_PLUGIN_REPOS); do \ - repo=$$(echo "$$entry" | sed 's/@[^@]*$$//'); \ - name=$$(basename $$repo .git); \ - plugin_dir="$(PLUGINS_CACHE)/$$name"; \ - schema_dir="$$plugin_dir/schema/pkl"; \ - if [ -d "$$schema_dir" ] && [ -f "$$schema_dir/PklProject" ]; then \ - plugin_name=$$(pkl eval -x 'name' "$$plugin_dir/formae-plugin.pkl"); \ - version=$$(pkl eval -x 'version' "$$plugin_dir/formae-plugin.pkl"); \ - echo "Publishing PKL schema for $$plugin_name@$$version..."; \ - aws s3 sync ".out/$${plugin_name}@$${version}" \ - "s3://hub.platform.engineering/plugins/$${plugin_name}/schema/pkl/$${plugin_name}/"; \ - fi \ - done - run: go run cmd/formae/main.go @@ -353,11 +164,14 @@ test-unit-summary: test-integration: go test -tags=integration -failfast ./... -test-e2e: build install-external-plugins +test-e2e: build echo "Setting up e2e PKL dependencies..." bash ./tests/e2e/go/setup_pkl.sh + echo "Staging formae binary in installer-shaped tree (.../bin/formae + .../.ops)..." + mkdir -p $(CURDIR)/dist/e2e/bin $(CURDIR)/dist/e2e/.ops + cp $(CURDIR)/formae $(CURDIR)/dist/e2e/bin/formae echo "Running e2e tests..." - E2E_FORMAE_BINARY=$(CURDIR)/formae go test -C ./tests/e2e/go -tags=e2e -timeout 30m -v ./... $(E2E_RUN_FLAGS) + E2E_FORMAE_BINARY=$(CURDIR)/dist/e2e/bin/formae go test -C ./tests/e2e/go -tags=e2e -timeout 30m -v ./... $(E2E_RUN_FLAGS) ## test-property: Run property tests (FullChaos 100 iterations, others 50) test-property: @@ -430,4 +244,4 @@ add-license: all: clean build gen-pkl api-docs -.PHONY: api-docs clean build install-gremlins build-debug fetch-external-plugins build-external-plugins install-external-plugins pkg-bin publish-bin gen-pkl gen-external-pkl pkg-pkl pkg-external-pkl publish-pkl publish-external-pkl run tidy-all test-build test-all test-unit test-unit-postgres test-unit-auroradataapi test-unit-summary test-integration test-e2e test-property mutation-test test-descriptors-pkl verify-schema-fakeaws version full-e2e lint lint-reuse add-license postgres-up postgres-down local-data-api-up local-data-api-down all \ No newline at end of file +.PHONY: api-docs clean build install-gremlins build-debug pkg-bin publish-bin gen-pkl pkg-pkl publish-pkl run tidy-all test-build test-all test-unit test-unit-postgres test-unit-auroradataapi test-unit-summary test-integration test-e2e test-property mutation-test test-descriptors-pkl verify-schema-fakeaws version full-e2e lint lint-reuse add-license postgres-up postgres-down local-data-api-up local-data-api-down all \ No newline at end of file diff --git a/docs/docs.go b/docs/docs.go index eb998bdd..ff80eebd 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -774,6 +774,9 @@ const docTemplate = `{ "model.FieldHint": { "type": "object", "properties": { + "AttachesTo": { + "type": "boolean" + }, "CreateOnly": { "type": "boolean" }, @@ -814,6 +817,19 @@ const docTemplate = `{ "FieldUpdateMethodNone" ] }, + "model.FilterCondition": { + "type": "object", + "properties": { + "propertyPath": { + "description": "PropertyPath is a JSONPath expression to query resource properties.\nExamples:\n - \"$.Tags[?(@.Key=='Name')].Value\" - get value of tag with key \"Name\"\n - \"$.Tags[?(@.Key=~'eks:automode:.*')]\" - check if any tag key matches regex\n - \"$.SkipDiscovery\" - get top-level property value", + "type": "string" + }, + "propertyValue": { + "description": "PropertyValue is the expected value to match.\nEmpty string means existence check (path returns any value = match).\nNon-empty means exact string match against the query result.", + "type": "string" + } + } + }, "model.ForceCheckTTLResponse": { "type": "object", "properties": { @@ -884,6 +900,22 @@ const docTemplate = `{ } } }, + "model.LabelConfig": { + "type": "object", + "properties": { + "defaultQuery": { + "description": "DefaultQuery is a JSONPath expression applied to all resources in this namespace.\nExample for AWS: ` + "`" + `$.Tags[?(@.Key=='Name')].Value` + "`" + `\nIf empty, falls back to NativeID.", + "type": "string" + }, + "resourceOverrides": { + "description": "ResourceOverrides provides JSONPath expressions for specific resource types,\noverriding the DefaultQuery. Use for resources without tags or with\nnon-standard label sources.\nKey: resource type (e.g., \"AWS::IAM::Policy\")\nValue: JSONPath expression (e.g., \"$.PolicyName\")", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, "model.ListCommandStatusResponse": { "type": "object", "properties": { @@ -895,6 +927,25 @@ const docTemplate = `{ } } }, + "model.MatchFilter": { + "type": "object", + "properties": { + "conditions": { + "description": "All conditions must match (AND logic) to exclude", + "type": "array", + "items": { + "$ref": "#/definitions/model.FilterCondition" + } + }, + "resourceTypes": { + "description": "Resource types this filter applies to", + "type": "array", + "items": { + "type": "string" + } + } + } + }, "model.ModifiedStack": { "type": "object", "properties": { @@ -909,6 +960,15 @@ const docTemplate = `{ "model.PluginInfo": { "type": "object", "properties": { + "DiscoveryFilters": { + "type": "array", + "items": { + "$ref": "#/definitions/model.MatchFilter" + } + }, + "LabelConfig": { + "$ref": "#/definitions/model.LabelConfig" + }, "MaxRequestsPerSecond": { "type": "integer" }, @@ -921,6 +981,15 @@ const docTemplate = `{ "ResourceCount": { "type": "integer" }, + "ResourceTypesToDiscover": { + "type": "array", + "items": { + "type": "string" + } + }, + "RetryConfig": { + "$ref": "#/definitions/model.RetryConfig" + }, "Version": { "type": "string" } @@ -1100,6 +1169,13 @@ const docTemplate = `{ "CascadeSource": { "type": "string" }, + "CreateOnlyPatch": { + "description": "CreateOnlyPatch is a JSON-patch document (same format as PatchDocument)\nlisting only the ops against createOnly fields that triggered a\nresource replacement. Populated on the delete half of a replace pair\nso the CLI can render which immutable properties forced the replace.\nNever sent to resource plugins — the replace executes as a plain\ndestroy + create.", + "type": "array", + "items": { + "type": "integer" + } + }, "CurrentAttempt": { "type": "integer" }, @@ -1172,6 +1248,20 @@ const docTemplate = `{ } } }, + "model.RetryConfig": { + "type": "object", + "properties": { + "maxRetries": { + "type": "integer" + }, + "retryDelay": { + "$ref": "#/definitions/time.Duration" + }, + "statusCheckInterval": { + "$ref": "#/definitions/time.Duration" + } + } + }, "model.Schema": { "type": "object", "properties": { @@ -1432,6 +1522,30 @@ const docTemplate = `{ "type": "string" } } + }, + "time.Duration": { + "type": "integer", + "format": "int64", + "enum": [ + -9223372036854775808, + 9223372036854775807, + 1, + 1000, + 1000000, + 1000000000, + 60000000000, + 3600000000000 + ], + "x-enum-varnames": [ + "minDuration", + "maxDuration", + "Nanosecond", + "Microsecond", + "Millisecond", + "Second", + "Minute", + "Hour" + ] } } }` diff --git a/docs/swagger.json b/docs/swagger.json index e93a2d05..53fca337 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -768,6 +768,9 @@ "model.FieldHint": { "type": "object", "properties": { + "AttachesTo": { + "type": "boolean" + }, "CreateOnly": { "type": "boolean" }, @@ -808,6 +811,19 @@ "FieldUpdateMethodNone" ] }, + "model.FilterCondition": { + "type": "object", + "properties": { + "propertyPath": { + "description": "PropertyPath is a JSONPath expression to query resource properties.\nExamples:\n - \"$.Tags[?(@.Key=='Name')].Value\" - get value of tag with key \"Name\"\n - \"$.Tags[?(@.Key=~'eks:automode:.*')]\" - check if any tag key matches regex\n - \"$.SkipDiscovery\" - get top-level property value", + "type": "string" + }, + "propertyValue": { + "description": "PropertyValue is the expected value to match.\nEmpty string means existence check (path returns any value = match).\nNon-empty means exact string match against the query result.", + "type": "string" + } + } + }, "model.ForceCheckTTLResponse": { "type": "object", "properties": { @@ -878,6 +894,22 @@ } } }, + "model.LabelConfig": { + "type": "object", + "properties": { + "defaultQuery": { + "description": "DefaultQuery is a JSONPath expression applied to all resources in this namespace.\nExample for AWS: `$.Tags[?(@.Key=='Name')].Value`\nIf empty, falls back to NativeID.", + "type": "string" + }, + "resourceOverrides": { + "description": "ResourceOverrides provides JSONPath expressions for specific resource types,\noverriding the DefaultQuery. Use for resources without tags or with\nnon-standard label sources.\nKey: resource type (e.g., \"AWS::IAM::Policy\")\nValue: JSONPath expression (e.g., \"$.PolicyName\")", + "type": "object", + "additionalProperties": { + "type": "string" + } + } + } + }, "model.ListCommandStatusResponse": { "type": "object", "properties": { @@ -889,6 +921,25 @@ } } }, + "model.MatchFilter": { + "type": "object", + "properties": { + "conditions": { + "description": "All conditions must match (AND logic) to exclude", + "type": "array", + "items": { + "$ref": "#/definitions/model.FilterCondition" + } + }, + "resourceTypes": { + "description": "Resource types this filter applies to", + "type": "array", + "items": { + "type": "string" + } + } + } + }, "model.ModifiedStack": { "type": "object", "properties": { @@ -903,6 +954,15 @@ "model.PluginInfo": { "type": "object", "properties": { + "DiscoveryFilters": { + "type": "array", + "items": { + "$ref": "#/definitions/model.MatchFilter" + } + }, + "LabelConfig": { + "$ref": "#/definitions/model.LabelConfig" + }, "MaxRequestsPerSecond": { "type": "integer" }, @@ -915,6 +975,15 @@ "ResourceCount": { "type": "integer" }, + "ResourceTypesToDiscover": { + "type": "array", + "items": { + "type": "string" + } + }, + "RetryConfig": { + "$ref": "#/definitions/model.RetryConfig" + }, "Version": { "type": "string" } @@ -1094,6 +1163,13 @@ "CascadeSource": { "type": "string" }, + "CreateOnlyPatch": { + "description": "CreateOnlyPatch is a JSON-patch document (same format as PatchDocument)\nlisting only the ops against createOnly fields that triggered a\nresource replacement. Populated on the delete half of a replace pair\nso the CLI can render which immutable properties forced the replace.\nNever sent to resource plugins — the replace executes as a plain\ndestroy + create.", + "type": "array", + "items": { + "type": "integer" + } + }, "CurrentAttempt": { "type": "integer" }, @@ -1166,6 +1242,20 @@ } } }, + "model.RetryConfig": { + "type": "object", + "properties": { + "maxRetries": { + "type": "integer" + }, + "retryDelay": { + "$ref": "#/definitions/time.Duration" + }, + "statusCheckInterval": { + "$ref": "#/definitions/time.Duration" + } + } + }, "model.Schema": { "type": "object", "properties": { @@ -1426,6 +1516,30 @@ "type": "string" } } + }, + "time.Duration": { + "type": "integer", + "format": "int64", + "enum": [ + -9223372036854775808, + 9223372036854775807, + 1, + 1000, + 1000000, + 1000000000, + 60000000000, + 3600000000000 + ], + "x-enum-varnames": [ + "minDuration", + "maxDuration", + "Nanosecond", + "Microsecond", + "Millisecond", + "Second", + "Minute", + "Hour" + ] } } } \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index dc3cb6d6..2fafbcf8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -74,6 +74,8 @@ definitions: type: object model.FieldHint: properties: + AttachesTo: + type: boolean CreateOnly: type: boolean HasProviderDefault: @@ -103,6 +105,23 @@ definitions: - FieldUpdateMethodSet - FieldUpdateMethodAtomic - FieldUpdateMethodNone + model.FilterCondition: + properties: + propertyPath: + description: |- + PropertyPath is a JSONPath expression to query resource properties. + Examples: + - "$.Tags[?(@.Key=='Name')].Value" - get value of tag with key "Name" + - "$.Tags[?(@.Key=~'eks:automode:.*')]" - check if any tag key matches regex + - "$.SkipDiscovery" - get top-level property value + type: string + propertyValue: + description: |- + PropertyValue is the expected value to match. + Empty string means existence check (path returns any value = match). + Non-empty means exact string match against the query result. + type: string + type: object model.ForceCheckTTLResponse: properties: command_ids: @@ -149,6 +168,25 @@ definitions: $ref: '#/definitions/model.Target' type: array type: object + model.LabelConfig: + properties: + defaultQuery: + description: |- + DefaultQuery is a JSONPath expression applied to all resources in this namespace. + Example for AWS: `$.Tags[?(@.Key=='Name')].Value` + If empty, falls back to NativeID. + type: string + resourceOverrides: + additionalProperties: + type: string + description: |- + ResourceOverrides provides JSONPath expressions for specific resource types, + overriding the DefaultQuery. Use for resources without tags or with + non-standard label sources. + Key: resource type (e.g., "AWS::IAM::Policy") + Value: JSONPath expression (e.g., "$.PolicyName") + type: object + type: object model.ListCommandStatusResponse: properties: Commands: @@ -156,6 +194,19 @@ definitions: $ref: '#/definitions/github_com_platform-engineering-labs_formae_pkg_api_model.Command' type: array type: object + model.MatchFilter: + properties: + conditions: + description: All conditions must match (AND logic) to exclude + items: + $ref: '#/definitions/model.FilterCondition' + type: array + resourceTypes: + description: Resource types this filter applies to + items: + type: string + type: array + type: object model.ModifiedStack: properties: ModifiedResources: @@ -165,6 +216,12 @@ definitions: type: object model.PluginInfo: properties: + DiscoveryFilters: + items: + $ref: '#/definitions/model.MatchFilter' + type: array + LabelConfig: + $ref: '#/definitions/model.LabelConfig' MaxRequestsPerSecond: type: integer Namespace: @@ -173,6 +230,12 @@ definitions: type: string ResourceCount: type: integer + ResourceTypesToDiscover: + items: + type: string + type: array + RetryConfig: + $ref: '#/definitions/model.RetryConfig' Version: type: string type: object @@ -293,6 +356,17 @@ definitions: properties: CascadeSource: type: string + CreateOnlyPatch: + description: |- + CreateOnlyPatch is a JSON-patch document (same format as PatchDocument) + listing only the ops against createOnly fields that triggered a + resource replacement. Populated on the delete half of a replace pair + so the CLI can render which immutable properties forced the replace. + Never sent to resource plugins — the replace executes as a plain + destroy + create. + items: + type: integer + type: array CurrentAttempt: type: integer Duration: @@ -341,6 +415,15 @@ definitions: StateMessage: type: string type: object + model.RetryConfig: + properties: + maxRetries: + type: integer + retryDelay: + $ref: '#/definitions/time.Duration' + statusCheckInterval: + $ref: '#/definitions/time.Duration' + type: object model.Schema: properties: Discoverable: @@ -515,6 +598,27 @@ definitions: TargetLabel: type: string type: object + time.Duration: + enum: + - -9223372036854775808 + - 9223372036854775807 + - 1 + - 1000 + - 1000000 + - 1000000000 + - 60000000000 + - 3600000000000 + format: int64 + type: integer + x-enum-varnames: + - minDuration + - maxDuration + - Nanosecond + - Microsecond + - Millisecond + - Second + - Minute + - Hour host: localhost:8080 info: contact: {} diff --git a/go.mod b/go.mod index cefc1a78..c3e045a6 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,7 @@ require ( github.com/platform-engineering-labs/formae/pkg/plugin v0.0.0-00010101000000-000000000000 github.com/platform-engineering-labs/formae/tests/testcontrol v0.0.0-00010101000000-000000000000 github.com/platform-engineering-labs/jsonpatch v0.0.0-20260421221004-fb6f96b174b5 - github.com/platform-engineering-labs/orbital v0.1.36 + github.com/platform-engineering-labs/orbital v0.1.38 github.com/posthog/posthog-go v1.6.3 github.com/pressly/goose/v3 v3.26.0 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index 892318c1..fbb354cf 100644 --- a/go.sum +++ b/go.sum @@ -359,12 +359,10 @@ github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= -github.com/platform-engineering-labs/jsonpatch v0.0.0-20260319010158-0bb81e14e83f h1:ZVwQGlGa5zFLuyTYiDL57KMlZQWM5zqvZFhHA6tTBPQ= -github.com/platform-engineering-labs/jsonpatch v0.0.0-20260319010158-0bb81e14e83f/go.mod h1:pKZWs1WAxW55mYtE0kAtpvrIMPzaQhZoBWSKqUMndvY= github.com/platform-engineering-labs/jsonpatch v0.0.0-20260421221004-fb6f96b174b5 h1:WRE1Dm9G920XZ+QPQBmwwU52WDJFf0bXQ1nnFFxfXuc= github.com/platform-engineering-labs/jsonpatch v0.0.0-20260421221004-fb6f96b174b5/go.mod h1:pKZWs1WAxW55mYtE0kAtpvrIMPzaQhZoBWSKqUMndvY= -github.com/platform-engineering-labs/orbital v0.1.36 h1:nPMLxDbwDrjlhJoMQXAE86HtVhQHC2A6BJlWYmNM75c= -github.com/platform-engineering-labs/orbital v0.1.36/go.mod h1:wwVPqOmW5RO78dDzL/G5CN1Ioao0P/aUsOZVrPVx64s= +github.com/platform-engineering-labs/orbital v0.1.38 h1:YRAYL9OWb+uiJc/qlPCB1u5wYpT6VUdHMUOnl2oQUV8= +github.com/platform-engineering-labs/orbital v0.1.38/go.mod h1:wwVPqOmW5RO78dDzL/G5CN1Ioao0P/aUsOZVrPVx64s= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/internal/agent/agent.go b/internal/agent/agent.go index 3ab5c8fb..ce3442f5 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -21,6 +21,7 @@ import ( "github.com/platform-engineering-labs/formae/internal/imconc" "github.com/platform-engineering-labs/formae/internal/logging" "github.com/platform-engineering-labs/formae/internal/metastructure" + "github.com/platform-engineering-labs/formae/internal/metastructure/plugin_manager" _ "github.com/platform-engineering-labs/formae/internal/network/all" _ "github.com/platform-engineering-labs/formae/internal/schema/all" "github.com/platform-engineering-labs/formae/internal/util" @@ -194,9 +195,23 @@ func (a *Agent) Start() error { } } + // Construct the plugin manager before announcing startup. Plugin + // distribution is the only install/upgrade path in 0.85, so an agent + // without one is not useful — and CLI users (often on a different + // host than the agent) would only see opaque 503s instead of this + // log line. Fail loudly here so the operator fixes config or the + // orbital tree before retrying. + pm, err := plugin_manager.New(slog.Default(), a.cfg.Artifacts.Repositories, []string{devPluginDir, systemPluginDir}) + if err != nil { + slog.Error("Plugin manager initialization failed; refusing to start", "error", err) + return + } + slog.Info("Agent started") apiServer := api.NewServer(a.ctx, ms, authHandle, &a.cfg.Agent.Server, a.cfg.Network, metricsHandler) + apiServer.SetPluginManager(pm) + imwg.Add(apiServer) imwg.Go(func() { apiServer.Start() diff --git a/internal/api/client.go b/internal/api/client.go index 85cf5555..a87bee5c 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -615,3 +615,135 @@ func (c *Client) ForceCheckTTL() (*apimodel.ForceCheckTTLResponse, error) { return nil, fmt.Errorf("unexpected response code from the forma agent: %d - %s", resp.StatusCode(), resp.String()) } } + +func (c *Client) ListPlugins(scope string, query, category, pluginType, channel string) (*apimodel.ListPluginsResponse, error) { + req := c.resty.R() + params := map[string]string{"scope": scope} + if query != "" { + params["q"] = query + } + if category != "" { + params["category"] = category + } + if pluginType != "" { + params["type"] = pluginType + } + if channel != "" { + params["channel"] = channel + } + req.SetQueryParams(params) + + resp, err := req.Get(c.endpoint + "/api/v1/plugins") + if err != nil { + return nil, err + } + + //nolint:errcheck + defer resp.Body.Close() + + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + } + + var result apimodel.ListPluginsResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &result, nil +} + +func (c *Client) GetPlugin(name, channel string) (*apimodel.GetPluginResponse, error) { + r := c.resty.R() + if channel != "" { + r = r.SetQueryParam("channel", channel) + } + resp, err := r.Get(c.endpoint + "/api/v1/plugins/" + name) + if err != nil { + return nil, err + } + + //nolint:errcheck + defer resp.Body.Close() + + if resp.StatusCode() == http.StatusNotFound { + return nil, nil + } + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode()) + } + + var result apimodel.GetPluginResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &result, nil +} + +func (c *Client) InstallPlugins(req apimodel.InstallPluginsRequest) (*apimodel.InstallPluginsResponse, error) { + resp, err := c.resty.R(). + SetHeader("Content-Type", "application/json"). + SetBody(req). + Post(c.endpoint + "/api/v1/plugins/install") + if err != nil { + return nil, err + } + + //nolint:errcheck + defer resp.Body.Close() + + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("plugin install failed: %d - %s", resp.StatusCode(), resp.String()) + } + + var result apimodel.InstallPluginsResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &result, nil +} + +func (c *Client) UninstallPlugins(req apimodel.UninstallPluginsRequest) (*apimodel.UninstallPluginsResponse, error) { + resp, err := c.resty.R(). + SetHeader("Content-Type", "application/json"). + SetBody(req). + Post(c.endpoint + "/api/v1/plugins/uninstall") + if err != nil { + return nil, err + } + + //nolint:errcheck + defer resp.Body.Close() + + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("plugin uninstall failed: %d - %s", resp.StatusCode(), resp.String()) + } + + var result apimodel.UninstallPluginsResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &result, nil +} + +func (c *Client) UpgradePlugins(req apimodel.UpgradePluginsRequest) (*apimodel.UpgradePluginsResponse, error) { + resp, err := c.resty.R(). + SetHeader("Content-Type", "application/json"). + SetBody(req). + Post(c.endpoint + "/api/v1/plugins/upgrade") + if err != nil { + return nil, err + } + + //nolint:errcheck + defer resp.Body.Close() + + if resp.StatusCode() != http.StatusOK { + return nil, fmt.Errorf("plugin upgrade failed: %d - %s", resp.StatusCode(), resp.String()) + } + + var result apimodel.UpgradePluginsResponse + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return nil, fmt.Errorf("failed to decode response: %w", err) + } + return &result, nil +} diff --git a/internal/api/plugin_handlers.go b/internal/api/plugin_handlers.go new file mode 100644 index 00000000..b98d923a --- /dev/null +++ b/internal/api/plugin_handlers.go @@ -0,0 +1,210 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package api + +import ( + "net/http" + + "github.com/labstack/echo/v4" + + "github.com/platform-engineering-labs/formae/internal/metastructure/plugin_manager" + apimodel "github.com/platform-engineering-labs/formae/pkg/api/model" +) + +func (s *Server) requirePluginManager(c echo.Context) (*plugin_manager.PluginManager, error) { + if s.pluginManager == nil { + return nil, echo.NewHTTPError(http.StatusServiceUnavailable, "plugin manager not configured") + } + return s.pluginManager, nil +} + +func (s *Server) listPluginsHandler(c echo.Context) error { + pm, err := s.requirePluginManager(c) + if err != nil { + return err + } + + scope := c.QueryParam("scope") + if scope == "" { + scope = "installed" + } + + var plugins []plugin_manager.Plugin + switch scope { + case "installed": + plugins, err = pm.List() + case "available": + plugins, err = pm.Available(plugin_manager.AvailableFilter{ + Query: c.QueryParam("q"), + Category: c.QueryParam("category"), + Type: c.QueryParam("type"), + Channel: c.QueryParam("channel"), + }) + default: + return echo.NewHTTPError(http.StatusBadRequest, "invalid scope: must be 'installed' or 'available'") + } + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + + return c.JSON(http.StatusOK, apimodel.ListPluginsResponse{ + Plugins: toAPIPlugins(plugins), + }) +} + +func (s *Server) getPluginHandler(c echo.Context) error { + pm, err := s.requirePluginManager(c) + if err != nil { + return err + } + + name := c.Param("name") + channel := c.QueryParam("channel") + p, err := pm.Info(name, channel) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + if p == nil { + return apiError(c, http.StatusNotFound, apimodel.PluginNotFound, apimodel.PluginNotFoundError{Name: name}) + } + return c.JSON(http.StatusOK, apimodel.GetPluginResponse{ + Plugin: toAPIPlugin(*p), + }) +} + +func (s *Server) installPluginsHandler(c echo.Context) error { + pm, err := s.requirePluginManager(c) + if err != nil { + return err + } + + var req apimodel.InstallPluginsRequest + if err := c.Bind(&req); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "invalid request body") + } + if len(req.Packages) == 0 { + return echo.NewHTTPError(http.StatusBadRequest, "packages list is required") + } + + pmReq := plugin_manager.InstallRequest{ + Packages: toManagerPackageRefs(req.Packages), + Channel: req.Channel, + } + resp, err := pm.Install(pmReq) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + return c.JSON(http.StatusOK, apimodel.InstallPluginsResponse{ + Operations: toAPIOperations(resp.Operations), + RequiresRestart: resp.RequiresRestart, + Warnings: resp.Warnings, + }) +} + +func (s *Server) uninstallPluginsHandler(c echo.Context) error { + pm, err := s.requirePluginManager(c) + if err != nil { + return err + } + + var req apimodel.UninstallPluginsRequest + if err := c.Bind(&req); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "invalid request body") + } + if len(req.Packages) == 0 { + return echo.NewHTTPError(http.StatusBadRequest, "packages list is required") + } + + pmReq := plugin_manager.UninstallRequest{ + Packages: toManagerPackageRefs(req.Packages), + } + resp, err := pm.Uninstall(pmReq) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + return c.JSON(http.StatusOK, apimodel.UninstallPluginsResponse{ + Operations: toAPIOperations(resp.Operations), + RequiresRestart: resp.RequiresRestart, + Warnings: resp.Warnings, + }) +} + +func (s *Server) upgradePluginsHandler(c echo.Context) error { + pm, err := s.requirePluginManager(c) + if err != nil { + return err + } + + var req apimodel.UpgradePluginsRequest + if err := c.Bind(&req); err != nil { + return echo.NewHTTPError(http.StatusBadRequest, "invalid request body") + } + + pmReq := plugin_manager.UpgradeRequest{ + Packages: toManagerPackageRefs(req.Packages), + Channel: req.Channel, + } + resp, err := pm.Upgrade(pmReq) + if err != nil { + return echo.NewHTTPError(http.StatusInternalServerError, err.Error()) + } + return c.JSON(http.StatusOK, apimodel.UpgradePluginsResponse{ + Operations: toAPIOperations(resp.Operations), + RequiresRestart: resp.RequiresRestart, + Warnings: resp.Warnings, + }) +} + +// Conversion helpers + +func toAPIPlugins(plugins []plugin_manager.Plugin) []apimodel.Plugin { + result := make([]apimodel.Plugin, 0, len(plugins)) + for _, p := range plugins { + result = append(result, toAPIPlugin(p)) + } + return result +} + +func toAPIPlugin(p plugin_manager.Plugin) apimodel.Plugin { + return apimodel.Plugin{ + Name: p.Name, + Kind: p.Kind, + Type: p.Type, + Namespace: p.Namespace, + Category: p.Category, + Summary: p.Summary, + Description: p.Description, + Publisher: p.Publisher, + License: p.License, + InstalledVersion: p.InstalledVersion, + AvailableVersions: p.AvailableVersions, + LocalPath: p.LocalPath, + Channel: p.Channel, + Frozen: p.Frozen, + ManagedBy: p.ManagedBy, + Metadata: p.Metadata, + } +} + +func toAPIOperations(ops []plugin_manager.Operation) []apimodel.PluginOperation { + result := make([]apimodel.PluginOperation, 0, len(ops)) + for _, op := range ops { + result = append(result, apimodel.PluginOperation{ + Name: op.Name, + Type: op.Type, + Version: op.Version, + Action: op.Action, + }) + } + return result +} + +func toManagerPackageRefs(refs []apimodel.PackageRef) []plugin_manager.PackageRef { + result := make([]plugin_manager.PackageRef, 0, len(refs)) + for _, r := range refs { + result = append(result, plugin_manager.PackageRef{Name: r.Name, Version: r.Version}) + } + return result +} diff --git a/internal/api/server.go b/internal/api/server.go index 433b2426..e6bd3502 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -26,6 +26,7 @@ import ( "github.com/platform-engineering-labs/formae/internal/logging" "github.com/platform-engineering-labs/formae/internal/metastructure" "github.com/platform-engineering-labs/formae/internal/metastructure/config" + "github.com/platform-engineering-labs/formae/internal/metastructure/plugin_manager" "github.com/platform-engineering-labs/formae/internal/network" apimodel "github.com/platform-engineering-labs/formae/pkg/api/model" pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" @@ -51,6 +52,12 @@ const ( DiscoverRoute = AdminBasePath + "/discover" CheckTTLRoute = AdminBasePath + "/check-ttl" + PluginsRoute = BasePath + "/plugins" + PluginRoute = BasePath + "/plugins/:name" + PluginInstallRoute = BasePath + "/plugins/install" + PluginUninstallRoute = BasePath + "/plugins/uninstall" + PluginUpgradeRoute = BasePath + "/plugins/upgrade" + HealthRoute = BasePath + "/health" MetricsRoute = "/metrics" APIDocsRoute = "/swagger/*" @@ -59,6 +66,7 @@ const ( type Server struct { echo *echo.Echo metastructure metastructure.MetastructureAPI + pluginManager *plugin_manager.PluginManager // nil if not configured ctx context.Context authHandle *auth.AuthPluginHandle serverConfig *pkgmodel.ServerConfig @@ -66,6 +74,13 @@ type Server struct { metricsHandler http.Handler } +// SetPluginManager configures the optional plugin manager for the server. +// When set, the plugin management REST endpoints become active; otherwise +// they return 503 Service Unavailable. +func (s *Server) SetPluginManager(pm *plugin_manager.PluginManager) { + s.pluginManager = pm +} + func NewServer(ctx context.Context, metastructure metastructure.MetastructureAPI, authHandle *auth.AuthPluginHandle, serverConfig *pkgmodel.ServerConfig, networkConfig *pkgmodel.NetworkConfig, metricsHandler http.Handler) *Server { server := &Server{ metastructure: metastructure, @@ -209,6 +224,13 @@ func (s *Server) configureEcho() *echo.Echo { e.POST(DiscoverRoute, s.ForceDiscover) e.POST(CheckTTLRoute, s.ForceCheckTTL) + // Plugin management endpoints + e.GET(PluginsRoute, s.listPluginsHandler) + e.POST(PluginInstallRoute, s.installPluginsHandler) + e.POST(PluginUninstallRoute, s.uninstallPluginsHandler) + e.POST(PluginUpgradeRoute, s.upgradePluginsHandler) + e.GET(PluginRoute, s.getPluginHandler) + // Prometheus metrics endpoint (if enabled) if s.metricsHandler != nil { e.GET(MetricsRoute, echo.WrapHandler(s.metricsHandler)) diff --git a/internal/cli/app/app.go b/internal/cli/app/app.go index abf7111a..2a814a58 100644 --- a/internal/cli/app/app.go +++ b/internal/cli/app/app.go @@ -50,6 +50,16 @@ func (a *App) Close() { } } +// NewClient creates a new API client using the App's configuration, +// auth, and network settings. +func (a *App) NewClient() (*api.Client, error) { + auth, net, err := a.getAuthAndNetHandlers() + if err != nil { + return nil, err + } + return api.NewClient(a.Config.Cli.API, auth, net), nil +} + type Plugins struct{} type Projects struct{} @@ -62,7 +72,10 @@ func NewApp() *App { } app := &App{ - Config: &pkgmodel.Config{}, + // Default PluginDir matches the PKL Config.pkl default so that CLI + // commands invoked without --config still get sane plugin discovery. + // LoadConfig overwrites this when a config file is present. + Config: &pkgmodel.Config{PluginDir: "~/.pel/formae/plugins"}, Plugins: Plugins{}, Projects: Projects{}, Usage: u, @@ -375,6 +388,115 @@ func (a *App) ForceDiscover() error { return client.ForceDiscover() } +func (a *App) InstallPlugins(req apimodel.InstallPluginsRequest) (*apimodel.InstallPluginsResponse, error) { + auth, net, err := a.getAuthAndNetHandlers() + if err != nil { + return nil, err + } + client := api.NewClient(a.Config.Cli.API, auth, net) + + if compatible, _, _, err := a.runBeforeCommand(client, true); !compatible { + return nil, err + } + + return client.InstallPlugins(req) +} + +func (a *App) UninstallPlugins(req apimodel.UninstallPluginsRequest) (*apimodel.UninstallPluginsResponse, error) { + auth, net, err := a.getAuthAndNetHandlers() + if err != nil { + return nil, err + } + client := api.NewClient(a.Config.Cli.API, auth, net) + + if compatible, _, _, err := a.runBeforeCommand(client, true); !compatible { + return nil, err + } + + return client.UninstallPlugins(req) +} + +// InstalledResourcePluginVersions queries the agent for installed resource +// plugins and returns a map of lowercase namespace to installed version. Used +// by `formae extract` and `formae project init` to pin remote schema URIs +// without scanning local plugin directories — orbital-installed plugins live +// on the agent box, not on the CLI box, so the local-scan approach broke for +// any deployment where agent and CLI are separate. +func (a *App) InstalledResourcePluginVersions() (map[string]string, error) { + plugins, err := a.installedResourcePlugins() + if err != nil { + return nil, err + } + result := make(map[string]string, len(plugins)) + for ns, info := range plugins { + if info.Version != "" { + result[ns] = info.Version + } + } + return result, nil +} + +// PluginInfo is a CLI-side view of an installed plugin, combining the +// agent-reported version with its on-disk PklProject location (when the +// agent and CLI share a filesystem). Used by the --schema-location local +// flow to build local PKL import strings. +type PluginInfo struct { + Version string + LocalPath string +} + +// InstalledResourcePlugins returns the agent's view of installed +// resource plugins, keyed by lowercase namespace (falling back to +// lowercase name when namespace is empty). Includes both version and +// the agent-reported on-disk PklProject path so callers can pick +// local vs remote URI emission. +func (a *App) InstalledResourcePlugins() (map[string]PluginInfo, error) { + return a.installedResourcePlugins() +} + +func (a *App) installedResourcePlugins() (map[string]PluginInfo, error) { + auth, net, err := a.getAuthAndNetHandlers() + if err != nil { + return nil, err + } + client := api.NewClient(a.Config.Cli.API, auth, net) + + resp, err := client.ListPlugins("installed", "", "", "", "") + if err != nil { + return nil, err + } + + result := make(map[string]PluginInfo, len(resp.Plugins)) + for _, p := range resp.Plugins { + if p.Type != "resource" { + continue + } + key := strings.ToLower(p.Namespace) + if key == "" { + key = strings.ToLower(p.Name) + } + result[key] = PluginInfo{ + Version: p.InstalledVersion, + LocalPath: p.LocalPath, + } + } + return result, nil +} + +func (a *App) UpgradePlugins(req apimodel.UpgradePluginsRequest) (*apimodel.UpgradePluginsResponse, error) { + auth, net, err := a.getAuthAndNetHandlers() + if err != nil { + return nil, err + } + client := api.NewClient(a.Config.Cli.API, auth, net) + + if compatible, _, _, err := a.runBeforeCommand(client, true); !compatible { + return nil, err + } + + return client.UpgradePlugins(req) +} + func (a *App) Stats() (*apimodel.Stats, []string, error) { auth, net, err := a.getAuthAndNetHandlers() if err != nil { @@ -537,18 +659,94 @@ func (a *App) SerializeForma(forma *pkgmodel.Forma, options *schema.SerializeOpt return "", err } + deps, err := a.buildDependencyStrings(forma, options.SchemaLocation) + if err != nil { + return "", err + } + if options.SchemaLocation == "" { + options.SchemaLocation = schema.SchemaLocationRemote + } + options.Dependencies = deps + return schemaPlugin.SerializeForma(forma, options) } -func (a *App) GenerateSourceCode(forma *pkgmodel.Forma, targetPath string, outputSchema string) (schema.GenerateSourcesResult, error) { +func (a *App) GenerateSourceCode(forma *pkgmodel.Forma, targetPath string, outputSchema string, schemaLocation schema.SchemaLocation) (schema.GenerateSourcesResult, error) { schemaPlugin, err := schema.DefaultRegistry.Get(outputSchema) if err != nil { return schema.GenerateSourcesResult{}, err } - // Extract always uses local schema resolution - includes, _ := a.Projects.formatIncludes(outputSchema, []string{"aws@local"}) - return schemaPlugin.GenerateSourceCode(forma, targetPath, includes, schema.SchemaLocationLocal) + deps, err := a.buildDependencyStrings(forma, schemaLocation) + if err != nil { + return schema.GenerateSourcesResult{}, err + } + if schemaLocation == "" { + schemaLocation = schema.SchemaLocationRemote + } + + options := &schema.SerializeOptions{ + Schema: outputSchema, + SchemaLocation: schemaLocation, + Dependencies: deps, + } + return schemaPlugin.GenerateSourceCode(forma, targetPath, nil, options) +} + +// buildDependencyStrings asks the agent for installed plugin info and +// emits PklProjectTemplate-formatted dep strings for every namespace +// present in the forma, plus formae core. +// +// SchemaLocationRemote (default) emits `.@` strings; +// PKL fetches these from hub.platform.engineering. SchemaLocationLocal +// emits `local::` strings pointing at the agent's on-disk +// PklProject; PKL imports them directly. Formae core is always remote +// (the agent does not surface its own PKL schema as a local path). +// +// SchemaLocationLocal requires the CLI and agent to share a filesystem. +// Each agent-reported localPath is statted; the first unreadable path +// (or first plugin missing from the agent's local view entirely) fails +// the call with a clear error pointing the operator at the same-box +// constraint. +func (a *App) buildDependencyStrings(forma *pkgmodel.Forma, location schema.SchemaLocation) ([]string, error) { + plugins, err := a.InstalledResourcePlugins() + if err != nil { + return nil, fmt.Errorf("listing installed plugins: %w", err) + } + + var deps []string + if formae.Version != "0.0.0" { + deps = append(deps, "pkl.formae@"+formae.Version) + } + + seen := make(map[string]bool) + for _, r := range forma.Resources { + ns := strings.ToLower(r.Namespace()) + if ns == "" || seen[ns] { + continue + } + seen[ns] = true + + info, ok := plugins[ns] + if !ok || info.Version == "" { + continue + } + + if location == schema.SchemaLocationLocal { + if info.LocalPath == "" { + return nil, fmt.Errorf("--schema-location local requires plugin %q to be installed on the agent's local filesystem; the agent reports no on-disk path. Install with `formae plugin install %s` and retry, or omit --schema-location to use remote schemas", ns, ns) + } + if _, statErr := os.Stat(info.LocalPath); statErr != nil { + return nil, fmt.Errorf("--schema-location local requires the CLI and agent to share a filesystem; the agent reports plugin %q at %s but that path is not readable from the CLI host (%v). Run the CLI on the agent's host, or omit --schema-location to use remote schemas", ns, info.LocalPath, statErr) + } + deps = append(deps, fmt.Sprintf("local:%s:%s", ns, info.LocalPath)) + } else { + deps = append(deps, fmt.Sprintf("%s.%s@%s", ns, ns, info.Version)) + } + } + + sort.Strings(deps) + return deps, nil } func (a *App) ExtractTargets(query string) ([]*pkgmodel.Target, []string, error) { @@ -631,12 +829,12 @@ func (p *Plugins) SupportedSchemas() []string { // Projects -func (p *Projects) Init(path string, format string, include []string) error { +func (p *Projects) Init(path string, format string, include []string, pluginsDir string, installedVersions map[string]string) error { // TODO(discount-elf) think about this namespace issue, since different packages can be included in plugins we currently // need plugin.package for download delivery switch format { case "pkl": - includes, err := p.formatIncludes(format, include) + includes, err := p.formatIncludes(format, include, pluginsDir, installedVersions) if err != nil { return err } @@ -672,7 +870,7 @@ func (p *Projects) Init(path string, format string, include []string) error { return nil } -func (p *Projects) formatIncludes(format string, include []string) ([]string, error) { +func (p *Projects) formatIncludes(format string, include []string, pluginsDir string, installedVersions map[string]string) ([]string, error) { var includes []string switch format { case "pkl": @@ -685,25 +883,27 @@ func (p *Projects) formatIncludes(format string, include []string) ([]string, er for _, inc := range include { ns, isLocal := parseIncludeSpec(inc) - // Find installed plugin info (handles case-insensitive lookup) - localPath, installedVersion := p.findInstalledPlugin(ns) - - // If @local suffix specified, must resolve locally + // @local: must resolve locally — pluginsDir is the dev plugin + // install dir (typically ~/.pel/formae/plugins, populated by + // `make install` in plugin repos). if isLocal { + localPath, _ := p.findInstalledPlugin(ns, pluginsDir) if localPath == "" { - return nil, fmt.Errorf("plugin %q not installed locally. Install with: formae plugin install %s", ns, ns) + return nil, fmt.Errorf("plugin %q not installed locally for @local resolution. Install it from a plugin repo with `make install`", ns) } includes = append(includes, fmt.Sprintf("local:%s:%s", ns, localPath)) continue } - // Default: resolve from hub (remote) - if installedVersion != "" { - includes = append(includes, fmt.Sprintf("%s.%s@%s", ns, ns, installedVersion)) - } else { - // No version info available, add as plain namespace (will fail at resolve time) - includes = append(includes, ns) + // Default: resolve from hub (remote). Version comes from the + // agent's installed-plugins view rather than scanning local + // disk, since orbital-installed plugins live with the agent + // and may not be present on the CLI box. + version, ok := installedVersions[ns] + if !ok || version == "" { + return nil, fmt.Errorf("plugin %q not installed on the agent. Install it with: formae plugin install %s", ns, ns) } + includes = append(includes, fmt.Sprintf("%s.%s@%s", ns, ns, version)) } default: return nil, nil @@ -727,8 +927,10 @@ func parseIncludeSpec(include string) (namespace string, isLocal bool) { // It performs case-insensitive directory lookup. // Returns (schemaPath, version) where schemaPath is the path to PklProject (empty if no schema), // and version is the highest installed version (empty if plugin not installed). -func (p *Projects) findInstalledPlugin(namespace string) (schemaPath string, version string) { - pluginsDir := util.ExpandHomePath("~/.pel/formae/plugins") +func (p *Projects) findInstalledPlugin(namespace, pluginsDir string) (schemaPath string, version string) { + if pluginsDir == "" { + return "", "" + } // Case-insensitive lookup: list plugins dir and find matching name pluginEntries, err := os.ReadDir(pluginsDir) diff --git a/internal/cli/eval/eval.go b/internal/cli/eval/eval.go index 00c8d508..607ca364 100644 --- a/internal/cli/eval/eval.go +++ b/internal/cli/eval/eval.go @@ -27,6 +27,7 @@ type EvalOptions struct { Beautify bool Colorize bool Properties map[string]string + SchemaLocation schema.SchemaLocation } func validateEvalOptions(opts *EvalOptions) error { @@ -65,6 +66,12 @@ func EvalCmd() *cobra.Command { opts.Beautify, _ = command.Flags().GetBool("beautify") opts.Colorize, _ = command.Flags().GetBool("colorize") opts.Properties = cmd.PropertiesFromCmd(command) + schemaLocation, _ := command.Flags().GetString("schema-location") + loc, err := parseSchemaLocation(schemaLocation) + if err != nil { + return err + } + opts.SchemaLocation = loc configFile, _ := command.Flags().GetString("config") app, err := cmd.AppFromContext(command.Context(), configFile, "", command) @@ -89,11 +96,25 @@ func EvalCmd() *cobra.Command { command.Flags().Bool("beautify", true, "beautify output (human consumer only)") command.Flags().Bool("colorize", true, "colorize output (human consumer only)") command.Flags().String("output-consumer", string(printer.ConsumerHuman), "Consumer of the command result (human | machine)") + command.Flags().String("schema-location", "remote", "How plugin PKL schemas are referenced when serializing the evaluated forma. 'remote' (default) emits package:// URIs that PKL fetches from the hub. 'local' emits local file imports against the agent's on-disk PklProject paths; requires CLI and agent to share a filesystem.") command.Flags().String("config", "", "Path to config file") return command } +// parseSchemaLocation maps the --schema-location flag value to the +// internal SchemaLocation enum. +func parseSchemaLocation(s string) (schema.SchemaLocation, error) { + switch s { + case "", "remote": + return schema.SchemaLocationRemote, nil + case "local": + return schema.SchemaLocationLocal, nil + default: + return "", cmd.FlagErrorf("invalid --schema-location %q; must be one of 'remote' or 'local'", s) + } +} + func runEval(app *app.App, opts *EvalOptions) error { if err := validateEvalOptions(opts); err != nil { return err @@ -113,9 +134,10 @@ func runEvalForHumans(app *app.App, opts *EvalOptions) error { return fmt.Errorf("cannot evaluate forma: %v", err) } output, err := app.SerializeForma(result, &schema.SerializeOptions{ - Schema: opts.OutputSchema, - Beautify: opts.Beautify, - Colorize: opts.Colorize, + Schema: opts.OutputSchema, + Beautify: opts.Beautify, + Colorize: opts.Colorize, + SchemaLocation: opts.SchemaLocation, }) if err != nil { return fmt.Errorf("cannot serialize eval result: %v", err) diff --git a/internal/cli/extract/extract.go b/internal/cli/extract/extract.go index 5f4cdd39..bc64f7fb 100644 --- a/internal/cli/extract/extract.go +++ b/internal/cli/extract/extract.go @@ -23,10 +23,11 @@ import ( ) type ExtractOptions struct { - TargetPath string - Query string - Yes bool - OutputSchema string + TargetPath string + Query string + Yes bool + OutputSchema string + SchemaLocation schema.SchemaLocation } func ExtractCmd() *cobra.Command { @@ -42,6 +43,12 @@ func ExtractCmd() *cobra.Command { opts.Query, _ = command.Flags().GetString("query") opts.Yes, _ = command.Flags().GetBool("yes") opts.OutputSchema, _ = command.Flags().GetString("output-schema") + schemaLocation, _ := command.Flags().GetString("schema-location") + loc, err := parseSchemaLocation(schemaLocation) + if err != nil { + return err + } + opts.SchemaLocation = loc configFile, _ := command.Flags().GetString("config") app, err := cmd.AppFromContext(command.Context(), configFile, "", command) @@ -64,11 +71,26 @@ func ExtractCmd() *cobra.Command { command.Flags().String("query", " ", "Query that allows to find resources by their attributes") command.Flags().Bool("yes", false, "Overwrite existing files without prompting") command.Flags().String("output-schema", "pkl", "Output schema (only 'pkl' is currently supported)") + command.Flags().String("schema-location", "remote", "How plugin PKL schemas are referenced in the generated PklProject. 'remote' (default) emits package:// URIs that PKL fetches from the hub. 'local' emits local file imports against the agent's on-disk PklProject paths; requires CLI and agent to share a filesystem.") command.Flags().String("config", "", "Path to config file") return command } +// parseSchemaLocation maps the --schema-location flag value to the +// internal SchemaLocation enum. Returns a clear error for unsupported +// values rather than silently accepting them. +func parseSchemaLocation(s string) (schema.SchemaLocation, error) { + switch s { + case "", "remote": + return schema.SchemaLocationRemote, nil + case "local": + return schema.SchemaLocationLocal, nil + default: + return "", cmd.FlagErrorf("invalid --schema-location %q; must be one of 'remote' or 'local'", s) + } +} + func runExtract(app *app.App, opts *ExtractOptions) error { err := validateExtractOptions(opts) if err != nil { @@ -108,7 +130,7 @@ func runExtract(app *app.App, opts *ExtractOptions) error { } } - res, err := app.GenerateSourceCode(forma, opts.TargetPath, opts.OutputSchema) + res, err := app.GenerateSourceCode(forma, opts.TargetPath, opts.OutputSchema, opts.SchemaLocation) if errors.Is(err, schema.ErrFailedToGenerateSources) { logFilePath := fmt.Sprintf("%s/log/client.log", config.Config.DataDirectory()) return fmt.Errorf("something went wrong during the extraction. This is our fault. Please contact us and send over the error logs from '%s'", logFilePath) diff --git a/internal/cli/plugin/install.go b/internal/cli/plugin/install.go new file mode 100644 index 00000000..6974b75f --- /dev/null +++ b/internal/cli/plugin/install.go @@ -0,0 +1,106 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin + +import ( + "fmt" + "log/slog" + "strings" + + "github.com/spf13/cobra" + + "github.com/platform-engineering-labs/formae/internal/cli/cmd" + "github.com/platform-engineering-labs/formae/internal/cli/display" + apimodel "github.com/platform-engineering-labs/formae/pkg/api/model" +) + +// parsePackageRefs parses "name" or "name@version" strings into PackageRefs. +func parsePackageRefs(args []string) []apimodel.PackageRef { + refs := make([]apimodel.PackageRef, 0, len(args)) + for _, arg := range args { + name, version, _ := strings.Cut(arg, "@") + refs = append(refs, apimodel.PackageRef{Name: name, Version: version}) + } + return refs +} + +// filterAuthOps returns the names of auth-type plugin operations. +func filterAuthOps(ops []apimodel.PluginOperation) []string { + var names []string + for _, op := range ops { + if op.Type == "auth" { + names = append(names, op.Name) + } + } + return names +} + +func PluginInstallCmd() *cobra.Command { + c := &cobra.Command{ + Use: "install [@]...", + Short: "Install plugins on the agent (and locally for auth plugins)", + Args: cobra.MinimumNArgs(1), + RunE: func(cc *cobra.Command, args []string) error { + channel, _ := cc.Flags().GetString("channel") + app, err := cmd.AppFromContext(cc.Context(), "", "", cc) + if err != nil { + return err + } + + req := apimodel.InstallPluginsRequest{ + Packages: parsePackageRefs(args), + Channel: channel, + } + resp, err := app.InstallPlugins(req) + if err != nil { + return err + } + + // Build the CLI-local manager BEFORE printing the agent install + // results. Constructing the local orbital manager can trigger + // sudo elevation when the binary lives at a privileged path + // (orbital.tree.New → InvokeSelfWithSudo → syscall.Exec). The + // re-execed process restarts from main and re-runs this whole + // command — printing the agent results before the elevation + // would produce duplicated lines. By deferring the prints + // until after the elevation has happened, the original process + // exits silently via syscall.Exec and only the privileged + // re-exec emits user-facing output. + authNames := filterAuthOps(resp.Operations) + var localMgr *CLIPluginManager + if len(authNames) > 0 { + localMgr, err = NewCLIPluginManager(slog.Default(), app.Config.Artifacts.Repositories) + if err != nil { + fmt.Printf(" %s CLI-side install failed: %s\n", display.Gold("!"), err.Error()) + fmt.Printf(" Retry with: formae plugin install %s\n", strings.Join(authNames, " ")) + return err + } + } + + for _, op := range resp.Operations { + fmt.Printf(" %s Installed %s %s on agent\n", display.Green("✓"), op.Name, op.Version) + } + + if localMgr != nil { + if localErr := localMgr.LocalInstall(authNames); localErr != nil { + fmt.Printf(" %s CLI-side install failed: %s\n", display.Gold("!"), localErr.Error()) + fmt.Printf(" Agent has the plugins. Retry with: formae plugin install %s\n", strings.Join(authNames, " ")) + return localErr + } + for _, name := range authNames { + fmt.Printf(" %s Installed %s on cli\n", display.Green("✓"), name) + } + } + + if resp.RequiresRestart { + fmt.Printf("\n %s Restart the agent to load the new plugins: formae agent restart\n", display.Gold("!")) + } + return nil + }, + SilenceErrors: true, + } + c.Flags().String("channel", "", "Install from a different channel") + return c +} diff --git a/internal/cli/plugin/manager.go b/internal/cli/plugin/manager.go new file mode 100644 index 00000000..0cca1f57 --- /dev/null +++ b/internal/cli/plugin/manager.go @@ -0,0 +1,55 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin + +import ( + "log/slog" + + "github.com/platform-engineering-labs/formae/internal/opsmgr" + pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" + "github.com/platform-engineering-labs/orbital/mgr" +) + +// CLIPluginManager manages plugin operations on the CLI host's local +// orbital tree. Used for dual-installing auth plugins that need to +// run on both the agent and the CLI. +type CLIPluginManager struct { + orb *mgr.Manager + logger *slog.Logger +} + +// NewCLIPluginManager creates a local plugin manager from the CLI's config. +// Returns nil, nil if no plugin repositories are configured. +func NewCLIPluginManager(logger *slog.Logger, repos []pkgmodel.Repository) (*CLIPluginManager, error) { + if len(repos) == 0 { + return nil, nil + } + orb, err := opsmgr.NewFromRepositories(logger, repos, "") + if err != nil { + return nil, err + } + return &CLIPluginManager{orb: orb, logger: logger}, nil +} + +// LocalInstall installs packages on the CLI host's orbital tree. +func (pm *CLIPluginManager) LocalInstall(names []string) error { + if err := pm.orb.Refresh(); err != nil { + pm.logger.Warn("failed to refresh local repos", "error", err) + } + return pm.orb.Install(names...) +} + +// LocalUninstall removes packages from the CLI host's orbital tree. +func (pm *CLIPluginManager) LocalUninstall(names []string) error { + return pm.orb.Remove(names...) +} + +// LocalUpgrade upgrades packages on the CLI host's orbital tree. +func (pm *CLIPluginManager) LocalUpgrade(names []string) error { + if err := pm.orb.Refresh(); err != nil { + pm.logger.Warn("failed to refresh local repos", "error", err) + } + return pm.orb.Update(names...) +} diff --git a/internal/cli/plugin/plugin.go b/internal/cli/plugin/plugin.go index a3497ec9..a78061dd 100644 --- a/internal/cli/plugin/plugin.go +++ b/internal/cli/plugin/plugin.go @@ -10,7 +10,6 @@ import ( "github.com/spf13/cobra" "github.com/platform-engineering-labs/formae/internal/cli/cmd" - "github.com/platform-engineering-labs/formae/internal/cli/display" ) func PluginCmd() *cobra.Command { @@ -19,12 +18,17 @@ func PluginCmd() *cobra.Command { Short: "Execute commands on plugins", Annotations: map[string]string{ "type": "Plugins", - "examples": "{{.Name}} {{.Command}} list\n{{.Name}} {{.Command}} init", + "examples": "{{.Name}} {{.Command}} list\n{{.Name}} {{.Command}} search aws\n{{.Name}} {{.Command}} install aws\n{{.Name}} {{.Command}} init", }, SilenceErrors: true, } command.AddCommand(PluginListCmd()) + command.AddCommand(PluginSearchCmd()) + command.AddCommand(PluginInfoCmd()) + command.AddCommand(PluginInstallCmd()) + command.AddCommand(PluginUninstallCmd()) + command.AddCommand(PluginUpgradeCmd()) command.AddCommand(PluginInitCmd()) command.SetUsageTemplate(cmd.SimpleCmdUsageTemplate) @@ -35,34 +39,100 @@ func PluginCmd() *cobra.Command { func PluginListCmd() *cobra.Command { command := &cobra.Command{ Use: "list", - Short: "List resource plugins registered with the agent", + Short: "List installed plugins", RunE: func(command *cobra.Command, args []string) error { app, err := cmd.AppFromContext(command.Context(), "", "", command) if err != nil { return err } - stats, _, err := app.Stats() + client, err := app.NewClient() if err != nil { return err } - if len(stats.Plugins) == 0 { - fmt.Println("No resource plugins registered with the agent.") - return nil + resp, err := client.ListPlugins("installed", "", "", "", "") + if err != nil { + return err + } + + fmt.Print(renderPluginList(resp.Plugins)) + return nil + }, + SilenceErrors: true, + } + return command +} + +func PluginSearchCmd() *cobra.Command { + c := &cobra.Command{ + Use: "search []", + Short: "Search available plugins", + RunE: func(cc *cobra.Command, args []string) error { + query := "" + if len(args) > 0 { + query = args[0] + } + category, _ := cc.Flags().GetString("category") + typ, _ := cc.Flags().GetString("type") + channel, _ := cc.Flags().GetString("channel") + + app, err := cmd.AppFromContext(cc.Context(), "", "", cc) + if err != nil { + return err } - fmt.Println(display.LightBlue("Resource Plugins")) - for _, p := range stats.Plugins { - fmt.Printf(" %s %s\n", - display.Green(p.Namespace), - display.Grey(p.Version)) + client, err := app.NewClient() + if err != nil { + return err + } + + resp, err := client.ListPlugins("available", query, category, typ, channel) + if err != nil { + return err } + fmt.Print(renderPluginSearch(resp.Plugins)) return nil }, SilenceErrors: true, } + c.Flags().String("category", "", "Filter by category") + c.Flags().String("type", "", "Filter by plugin type (resource|auth)") + c.Flags().String("channel", "", "Search a different channel") + return c +} - return command +func PluginInfoCmd() *cobra.Command { + c := &cobra.Command{ + Use: "info ", + Short: "Show detailed plugin information", + Args: cobra.ExactArgs(1), + RunE: func(cc *cobra.Command, args []string) error { + channel, _ := cc.Flags().GetString("channel") + app, err := cmd.AppFromContext(cc.Context(), "", "", cc) + if err != nil { + return err + } + + client, err := app.NewClient() + if err != nil { + return err + } + + resp, err := client.GetPlugin(args[0], channel) + if err != nil { + return err + } + if resp == nil { + return fmt.Errorf("plugin '%s' not found", args[0]) + } + + fmt.Print(renderPluginInfo(&resp.Plugin)) + return nil + }, + SilenceErrors: true, + } + c.Flags().String("channel", "", "Query a different channel") + return c } diff --git a/internal/cli/plugin/render.go b/internal/cli/plugin/render.go new file mode 100644 index 00000000..c54b46c4 --- /dev/null +++ b/internal/cli/plugin/render.go @@ -0,0 +1,137 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin + +import ( + "fmt" + "strings" + + "github.com/platform-engineering-labs/formae/internal/cli/display" + apimodel "github.com/platform-engineering-labs/formae/pkg/api/model" +) + +func renderPluginList(agentPlugins []apimodel.Plugin) string { + if len(agentPlugins) == 0 { + return "No plugins installed.\n" + } + + var sb strings.Builder + + // Group by kind/type. Bundles get their own section so users can see + // what curated collections they have installed alongside the + // individual plugins those bundles pulled in. Internally orbital + // calls these "metapackages"; we render them as "bundles" in the CLI + // for friendlier vocabulary. + var resource, auth, bundles []apimodel.Plugin + for _, p := range agentPlugins { + switch { + case p.Kind == "metapackage": + bundles = append(bundles, p) + case p.Type == "auth": + auth = append(auth, p) + default: + resource = append(resource, p) + } + } + + emit := func(header string, plugins []apimodel.Plugin, addLeadingNewline bool) { + if len(plugins) == 0 { + return + } + if addLeadingNewline { + sb.WriteString("\n") + } + sb.WriteString(display.LightBlue(header) + "\n") + for _, p := range plugins { + line := fmt.Sprintf(" %s %s %s", + display.Green("✓"), + padRight(p.Name, 14), + display.Grey(p.InstalledVersion)) + if p.ManagedBy != "" { + line += display.Grey(" (" + p.ManagedBy + ")") + } + sb.WriteString(line + "\n") + } + } + + emit("Resource plugins:", resource, false) + emit("Auth plugins:", auth, len(resource) > 0) + emit("Bundles:", bundles, len(resource) > 0 || len(auth) > 0) + + return sb.String() +} + +func renderPluginSearch(plugins []apimodel.Plugin) string { + if len(plugins) == 0 { + return "No plugins found.\n" + } + + var sb strings.Builder + for _, p := range plugins { + line := fmt.Sprintf(" %s %s", + padRight(p.Name, 14), + p.Summary) + if p.InstalledVersion != "" { + line += " " + display.Green("✓ installed") + } + sb.WriteString(line + "\n") + } + return sb.String() +} + +func renderPluginInfo(p *apimodel.Plugin) string { + var sb strings.Builder + installed := "" + if p.InstalledVersion != "" { + installed = " (installed)" + } + fmt.Fprintf(&sb, "%s %s%s\n", display.LightBlue(p.Name), p.InstalledVersion, display.Green(installed)) + + // For bundles display "bundle" rather than the empty plugin type and + // surface the description so the user understands what the bundle + // pulls in. For plugins, fall back to the plugin runtime type + // (resource | auth) and namespace as before. ("metapackage" is the + // internal orbital term; "bundle" is what we show users.) + if p.Kind == "metapackage" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Type:"), "bundle") + } else { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Type:"), p.Type) + if p.Namespace != "" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Namespace:"), p.Namespace) + } + } + if p.Category != "" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Category:"), p.Category) + } + if p.Summary != "" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Summary:"), p.Summary) + } + if p.Kind == "metapackage" && p.Description != "" && p.Description != p.Summary { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Description:"), p.Description) + } + if p.Publisher != "" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Publisher:"), p.Publisher) + } + if p.License != "" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("License:"), p.License) + } + if p.Channel != "" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Channel:"), p.Channel) + } + if len(p.AvailableVersions) > 0 { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Available:"), strings.Join(p.AvailableVersions, ", ")) + } + if p.ManagedBy != "" { + fmt.Fprintf(&sb, " %-14s %s\n", display.Grey("Part of:"), p.ManagedBy) + } + return sb.String() +} + +func padRight(s string, width int) string { + if len(s) >= width { + return s + } + return s + strings.Repeat(" ", width-len(s)) +} diff --git a/internal/cli/plugin/uninstall.go b/internal/cli/plugin/uninstall.go new file mode 100644 index 00000000..8b98e265 --- /dev/null +++ b/internal/cli/plugin/uninstall.go @@ -0,0 +1,71 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin + +import ( + "fmt" + "log/slog" + + "github.com/spf13/cobra" + + "github.com/platform-engineering-labs/formae/internal/cli/cmd" + "github.com/platform-engineering-labs/formae/internal/cli/display" + apimodel "github.com/platform-engineering-labs/formae/pkg/api/model" +) + +func PluginUninstallCmd() *cobra.Command { + c := &cobra.Command{ + Use: "uninstall ...", + Aliases: []string{"remove"}, + Short: "Uninstall plugins from the agent (and locally for auth plugins)", + Args: cobra.MinimumNArgs(1), + RunE: func(cc *cobra.Command, args []string) error { + app, err := cmd.AppFromContext(cc.Context(), "", "", cc) + if err != nil { + return err + } + + refs := make([]apimodel.PackageRef, 0, len(args)) + for _, name := range args { + refs = append(refs, apimodel.PackageRef{Name: name}) + } + + resp, err := app.UninstallPlugins(apimodel.UninstallPluginsRequest{Packages: refs}) + if err != nil { + return err + } + + // Build the CLI-local manager before printing the agent results + // to avoid duplicated output if orbital re-execs us under sudo. + // See install.go for the full explanation. + authNames := filterAuthOps(resp.Operations) + var localMgr *CLIPluginManager + if len(authNames) > 0 { + localMgr, _ = NewCLIPluginManager(slog.Default(), app.Config.Artifacts.Repositories) + } + + for _, op := range resp.Operations { + fmt.Printf(" %s Removed %s from agent\n", display.Green("✓"), op.Name) + } + + if localMgr != nil { + if localErr := localMgr.LocalUninstall(authNames); localErr != nil { + fmt.Printf(" %s CLI-side uninstall failed: %s\n", display.Gold("!"), localErr.Error()) + } else { + for _, name := range authNames { + fmt.Printf(" %s Removed %s from cli\n", display.Green("✓"), name) + } + } + } + + if resp.RequiresRestart { + fmt.Printf("\n %s Restart the agent to unload the plugins: formae agent restart\n", display.Gold("!")) + } + return nil + }, + SilenceErrors: true, + } + return c +} diff --git a/internal/cli/plugin/upgrade.go b/internal/cli/plugin/upgrade.go new file mode 100644 index 00000000..3b6404a7 --- /dev/null +++ b/internal/cli/plugin/upgrade.go @@ -0,0 +1,83 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin + +import ( + "fmt" + "log/slog" + "strings" + + "github.com/spf13/cobra" + + "github.com/platform-engineering-labs/formae/internal/cli/cmd" + "github.com/platform-engineering-labs/formae/internal/cli/display" + apimodel "github.com/platform-engineering-labs/formae/pkg/api/model" +) + +func PluginUpgradeCmd() *cobra.Command { + c := &cobra.Command{ + Use: "upgrade [[@]...]", + Short: "Upgrade plugins on the agent (and locally for auth plugins)", + Long: "Upgrade one or more plugins. If no arguments are given, all installed plugins are upgraded.", + RunE: func(cc *cobra.Command, args []string) error { + channel, _ := cc.Flags().GetString("channel") + app, err := cmd.AppFromContext(cc.Context(), "", "", cc) + if err != nil { + return err + } + + req := apimodel.UpgradePluginsRequest{ + Packages: parsePackageRefs(args), + Channel: channel, + } + resp, err := app.UpgradePlugins(req) + if err != nil { + return err + } + + if len(resp.Operations) == 0 { + fmt.Println(" All plugins are up to date.") + return nil + } + + // Build the CLI-local manager before printing the agent results + // to avoid duplicated output if orbital re-execs us under sudo. + // See install.go for the full explanation. + authNames := filterAuthOps(resp.Operations) + var localMgr *CLIPluginManager + if len(authNames) > 0 { + localMgr, err = NewCLIPluginManager(slog.Default(), app.Config.Artifacts.Repositories) + if err != nil { + fmt.Printf(" %s CLI-side upgrade failed: %s\n", display.Gold("!"), err.Error()) + fmt.Printf(" Retry with: formae plugin upgrade %s\n", strings.Join(authNames, " ")) + return err + } + } + + for _, op := range resp.Operations { + fmt.Printf(" %s Upgraded %s to %s on agent\n", display.Green("✓"), op.Name, op.Version) + } + + if localMgr != nil { + if localErr := localMgr.LocalUpgrade(authNames); localErr != nil { + fmt.Printf(" %s CLI-side upgrade failed: %s\n", display.Gold("!"), localErr.Error()) + fmt.Printf(" Agent has the updated plugins. Retry with: formae plugin upgrade %s\n", strings.Join(authNames, " ")) + return localErr + } + for _, name := range authNames { + fmt.Printf(" %s Upgraded %s on cli\n", display.Green("✓"), name) + } + } + + if resp.RequiresRestart { + fmt.Printf("\n %s Restart the agent to load the updated plugins: formae agent restart\n", display.Gold("!")) + } + return nil + }, + SilenceErrors: true, + } + c.Flags().String("channel", "", "Upgrade from a different channel") + return c +} diff --git a/internal/cli/project/project.go b/internal/cli/project/project.go index d5e4b7c5..390ec380 100644 --- a/internal/cli/project/project.go +++ b/internal/cli/project/project.go @@ -6,12 +6,15 @@ package project import ( "fmt" + "strings" "github.com/spf13/cobra" + "github.com/platform-engineering-labs/formae/internal/cli/app" "github.com/platform-engineering-labs/formae/internal/cli/cmd" "github.com/platform-engineering-labs/formae/internal/cli/display" "github.com/platform-engineering-labs/formae/internal/cli/prompter" + "github.com/platform-engineering-labs/formae/internal/util" ) func ProjectCmd() *cobra.Command { @@ -40,6 +43,7 @@ func ProjectInitCmd() *cobra.Command { schema, _ := command.Flags().GetString("schema") include, _ := command.Flags().GetStringArray("include") yes, _ := command.Flags().GetBool("yes") + pluginDir, _ := command.Flags().GetString("plugin-dir") // Confirm with user if no plugins specified if len(include) == 0 && !yes { @@ -55,12 +59,25 @@ func ProjectInitCmd() *cobra.Command { fmt.Println() } - app, err := cmd.AppFromContext(command.Context(), "", "", command) - if err != nil { - return err + // Non-@local includes need plugin versions from the agent — + // after the multi-source plugin-discovery refactor, plugins + // live on the agent box, not the CLI box. Skip the agent + // query if every include is @local (no version needed). + var installedVersions map[string]string + if needsAgent(include) { + configFile, _ := command.Flags().GetString("config") + appCtx, err := cmd.AppFromContext(command.Context(), configFile, "", command) + if err != nil { + return err + } + installedVersions, err = appCtx.InstalledResourcePluginVersions() + if err != nil { + return fmt.Errorf("listing installed plugins: %w", err) + } } - return app.Projects.Init(command.Flags().Arg(0), schema, include) + projects := &app.Projects{} + return projects.Init(command.Flags().Arg(0), schema, include, util.ExpandHomePath(pluginDir), installedVersions) }, SilenceErrors: true, } @@ -68,6 +85,20 @@ func ProjectInitCmd() *cobra.Command { command.Flags().String("schema", "pkl", "Schema to use for the project (pkl)") command.Flags().StringArray("include", []string{}, "Packages to include (use @local suffix for local plugins, e.g. myplugin@local)") command.Flags().BoolP("yes", "y", false, "Skip confirmation prompts") + command.Flags().String("plugin-dir", "~/.pel/formae/plugins", "Directory to scan for @local plugin schemas") + command.Flags().String("config", "", "Path to config file") return command } + +// needsAgent reports whether resolving the given include list requires asking +// the agent for installed plugin versions. Includes ending in @local resolve +// against pluginsDir on disk and don't need an agent. +func needsAgent(include []string) bool { + for _, inc := range include { + if !strings.HasSuffix(strings.ToLower(inc), "@local") { + return true + } + } + return false +} diff --git a/internal/cli/renderer/errors.go b/internal/cli/renderer/errors.go index fa092df1..2f2b2931 100644 --- a/internal/cli/renderer/errors.go +++ b/internal/cli/renderer/errors.go @@ -94,6 +94,14 @@ func RenderErrorMessage(err error) (string, error) { msg = renderNonPortableResourcesError(&errResp.Data) } + if errResp, ok := err.(*apimodel.ErrorResponse[apimodel.PluginNotFoundError]); ok { + msg = display.Redf("plugin '%s' not found\n", errResp.Data.Name) + } + + if errResp, ok := err.(*apimodel.ErrorResponse[apimodel.PluginDependencyConflictError]); ok { + msg = display.Redf("plugin dependency conflict: %s\n", errResp.Data.Message) + } + if msg == "" { return "", err } diff --git a/internal/cli/update/update.go b/internal/cli/update/update.go index b8c53780..9e19ac12 100644 --- a/internal/cli/update/update.go +++ b/internal/cli/update/update.go @@ -14,6 +14,8 @@ import ( "github.com/platform-engineering-labs/formae/internal/cli/config" "github.com/platform-engineering-labs/formae/internal/logging" "github.com/platform-engineering-labs/formae/internal/opsmgr" + pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" + "github.com/platform-engineering-labs/orbital/mgr" "github.com/platform-engineering-labs/orbital/opm/records" "github.com/platform-engineering-labs/orbital/ops" "github.com/spf13/cobra" @@ -41,7 +43,12 @@ func UpdateCmd() *cobra.Command { return err } - orb, err := opsmgr.New(slog.Default(), app.Config.Artifacts.URL, channel) + var orb *mgr.Manager + if len(app.Config.Artifacts.Repositories) > 0 { + orb, err = opsmgr.NewFromRepositoriesFiltered(slog.Default(), app.Config.Artifacts.Repositories, channel, pkgmodel.RepositoryTypeBinary) + } else { + orb, err = opsmgr.New(slog.Default(), app.Config.Artifacts.URL, channel) + } if err != nil { return err } @@ -143,7 +150,12 @@ func UpdateListCmd() *cobra.Command { return err } - orb, err := opsmgr.New(slog.Default(), app.Config.Artifacts.URL, channel) + var orb *mgr.Manager + if len(app.Config.Artifacts.Repositories) > 0 { + orb, err = opsmgr.NewFromRepositoriesFiltered(slog.Default(), app.Config.Artifacts.Repositories, channel, pkgmodel.RepositoryTypeBinary) + } else { + orb, err = opsmgr.New(slog.Default(), app.Config.Artifacts.URL, channel) + } if err != nil { return err } diff --git a/internal/metastructure/plugin_manager/helpers_test.go b/internal/metastructure/plugin_manager/helpers_test.go new file mode 100644 index 00000000..6747046c --- /dev/null +++ b/internal/metastructure/plugin_manager/helpers_test.go @@ -0,0 +1,26 @@ +//go:build unit + +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin_manager + +import "log/slog" + +// newForTesting creates a PluginManager whose factory returns the same fake +// for every channel. Use newForTestingWithFactory when a test needs to model +// per-channel behavior. +func newForTesting(logger *slog.Logger, orb orbitalClient) *PluginManager { + return newForTestingWithFactory(logger, orb, func(string) (orbitalClient, error) { + return orb, nil + }) +} + +// newForTestingWithFactory creates a PluginManager with an explicit factory +// for channel-specific tests. listOrb is used for List/Uninstall (channel- +// agnostic), while factory(channel) is consulted for Available/Info/Install/ +// Upgrade. +func newForTestingWithFactory(logger *slog.Logger, listOrb orbitalClient, factory orbitalFactory) *PluginManager { + return &PluginManager{logger: logger, listOrb: listOrb, factory: factory} +} diff --git a/internal/metastructure/plugin_manager/plugin_manager.go b/internal/metastructure/plugin_manager/plugin_manager.go new file mode 100644 index 00000000..24162236 --- /dev/null +++ b/internal/metastructure/plugin_manager/plugin_manager.go @@ -0,0 +1,529 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin_manager + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/platform-engineering-labs/formae/internal/opsmgr" + pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" + "github.com/platform-engineering-labs/formae/pkg/plugin/discovery" + "github.com/platform-engineering-labs/orbital/opm/records" + "github.com/platform-engineering-labs/orbital/ops" +) + +// orbitalClient abstracts the orbital Manager for testing. +type orbitalClient interface { + Refresh() error + Install(packages ...string) error + Remove(packages ...string) error + Update(packages ...string) error + Ready() bool + List() ([]*records.Package, error) + Available() (map[string]*records.Status, error) + AvailableFor(name string) (*records.Status, error) +} + +// Plugin describes a single plugin from the manager's perspective. +// +// Kind distinguishes regular plugins ("plugin") from curated bundles +// ("metapackage") and is taken straight from Metadata["display"]["kind"]. +// Type is only meaningful for kind=="plugin" and reflects the plugin's +// runtime role (resource | auth); it is empty for metapackages. +type Plugin struct { + Name string + Kind string // "plugin" | "metapackage" + Type string // "resource" | "auth" + Namespace string + Category string + Summary string + Description string + Publisher string + License string + InstalledVersion string + AvailableVersions []string + Channel string + Frozen bool + ManagedBy string // "standard" | "" + // LocalPath is the absolute path to the plugin's PklProject file on + // the agent's filesystem, populated from the discovery scan. Empty + // when no on-disk install is found (e.g. orbital recorded the + // package but the binary tree was wiped, or the plugin is only + // visible in an unscanned location). Surfaced via the API so the + // CLI can build local schema URIs for --schema-location local. + LocalPath string + Metadata map[string]map[string]string +} + +// AvailableFilter constrains the set of plugins returned by Available. +type AvailableFilter struct { + Query string + Category string + Type string + Channel string +} + +// PackageRef identifies a package by name and optional version. +type PackageRef struct { + Name string + Version string +} + +// Operation describes a single action taken on a package. +type Operation struct { + Name string + Type string + Version string + Action string // "install" | "remove" | "noop" +} + +// Response is the result of an Install, Uninstall, or Upgrade call. +type Response struct { + Operations []Operation + RequiresRestart bool + Warnings []string +} + +// InstallRequest is the input to Install. +type InstallRequest struct { + Packages []PackageRef + Channel string +} + +// UninstallRequest is the input to Uninstall. +type UninstallRequest struct{ Packages []PackageRef } + +// UpgradeRequest is the input to Upgrade. +type UpgradeRequest struct { + Packages []PackageRef + Channel string +} + +// DefaultChannel is the channel queried when none is specified explicitly. +// Operators publishing to a different channel must opt in via --channel. +const DefaultChannel = "stable" + +// orbitalFactory builds an orbitalClient for the requested channel. When the +// channel string is non-empty, the URI fragments of every configured repo are +// overridden with that channel before the manager is constructed. +type orbitalFactory func(channel string) (orbitalClient, error) + +// PluginManager manages plugin lifecycle operations via orbital. +// +// listOrb is the long-lived client used for inspecting locally-installed +// packages (List/Uninstall) — those operations are channel-agnostic. Per-call +// factory invocations build channel-specific clients for queries that depend +// on a remote channel (Available/Info/Install/Upgrade). +// +// pluginDirs are the directories the agent scans for on-disk plugin +// installs. The same dirs the agent scans at startup to populate the +// PluginProcessSupervisor; the manager re-scans them on List() to +// attach LocalPath to each plugin. Order matters: dirs earlier in the +// slice take priority when the same plugin is found in multiple dirs +// (mirrors discovery.DiscoverPluginsMulti). +type PluginManager struct { + logger *slog.Logger + listOrb orbitalClient + factory orbitalFactory + pluginDirs []string +} + +// New creates a PluginManager backed by the orbital repositories in repos. +// Both formae-plugin and binary repos are loaded (the solver needs both for +// cross-repo dependency resolution, e.g. plugin depends on formae >= 0.85); +// the per-package "is this a plugin?" classification is done at query time +// via Metadata["plugin"] so that binary-only packages (formae itself, pkl +// tooling, etc.) don't surface through List, Available, or Info. +// +// Returns an error if the underlying orbital tree is not initialized at the +// derived tree-root path. The agent treats this as a fatal startup error so +// the operator gets a clear log line rather than CLI users seeing opaque +// 503s on every plugin command. +func New(logger *slog.Logger, repos []pkgmodel.Repository, pluginDirs []string) (*PluginManager, error) { + factory := func(channel string) (orbitalClient, error) { + return opsmgr.NewFromRepositories(logger, repos, channel) + } + listOrb, err := factory("") + if err != nil { + return nil, err + } + if !listOrb.Ready() { + return nil, fmt.Errorf("orbital tree not initialized at the path derived from the formae binary location; install formae via the orbital installer or set up an orbital tree manually") + } + // Warm orbital's repository cache at startup. Without this, the first + // plugin install on a fresh tree fails with "no install candidates + // found" because the install endpoint doesn't auto-refresh — only + // Available does. Best-effort: a refresh failure (e.g. hub unreachable) + // must not block agent startup; subsequent operations will retry. + if err := listOrb.Refresh(); err != nil { + logger.Warn("orbital cache refresh failed at startup; plugin operations will rely on cached data until the next refresh", "error", err) + } + return &PluginManager{logger: logger, listOrb: listOrb, factory: factory, pluginDirs: pluginDirs}, nil +} + +// clientFor returns an orbitalClient configured for the given channel. An +// empty channel resolves to DefaultChannel so callers don't need to special- +// case the default. +func (pm *PluginManager) clientFor(channel string) (orbitalClient, error) { + if channel == "" { + channel = DefaultChannel + } + return pm.factory(channel) +} + +// isPluginPackage reports whether pkg is a formae plugin or a curated +// metapackage. The classification is purely metadata-driven via +// Metadata["display"]["kind"]: regular plugins set kind="plugin", curated +// bundles set kind="metapackage". Binary-only tooling (the formae binary +// itself, pkl) has no display.kind and is filtered out. +func isPluginPackage(pkg *records.Package) bool { + if pkg == nil || pkg.Header == nil { + return false + } + disp, ok := pkg.Metadata["display"] + if !ok { + return false + } + kind := disp["kind"] + return kind == "plugin" || kind == "metapackage" +} + +// versionString renders a Version as Major.Minor.Patch with PreRelease and +// Build appended in semver-2.0 form. Orbital's Version.Short calls Semver +// which drops PreRelease and Build, so e.g. 0.1.0-dev.1 would render as +// 0.1.0 — we always want the full identifier so users can tell channels +// apart at a glance. +func versionString(v *ops.Version) string { + if v == nil { + return "" + } + s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) + if v.PreRelease != "" { + s += "-" + v.PreRelease + } + if v.Build != "" { + s += "+" + v.Build + } + return s +} + +// List returns every installed plugin. Packages without "plugin" metadata +// (e.g. the formae binary itself, pkl tooling) are filtered out. +// +// Each plugin's LocalPath is populated from a fresh discovery scan of the +// configured pluginDirs. A plugin known to orbital but missing from the +// scan dirs (e.g. a record persisted before the binary tree moved) gets +// LocalPath="" — consumers should treat that as "no on-disk install +// available". +func (pm *PluginManager) List() ([]Plugin, error) { + pkgs, err := pm.listOrb.List() + if err != nil { + return nil, fmt.Errorf("listing installed packages: %w", err) + } + localPaths := pm.discoverLocalPaths() + plugins := make([]Plugin, 0, len(pkgs)) + for _, pkg := range pkgs { + if !isPluginPackage(pkg) { + continue + } + p := pluginFromPackage(pkg) + if pkg.Version != nil { + p.InstalledVersion = versionString(pkg.Version) + } + if path, ok := localPaths[strings.ToLower(p.Name)]; ok { + p.LocalPath = path + } + plugins = append(plugins, p) + } + return plugins, nil +} + +// discoverLocalPaths scans the configured plugin dirs and returns a map +// from lowercase plugin name to the absolute path of the plugin's +// PklProject file. Errors during scanning are logged and the partial +// result is returned; an empty map means no on-disk plugins were found +// (which is a valid state, not an error). +func (pm *PluginManager) discoverLocalPaths() map[string]string { + if len(pm.pluginDirs) == 0 { + return map[string]string{} + } + infos := discovery.DiscoverPluginsMulti(pm.pluginDirs, discovery.Resource) + authInfos := discovery.DiscoverPluginsMulti(pm.pluginDirs, discovery.Auth) + infos = append(infos, authInfos...) + out := make(map[string]string, len(infos)) + for _, info := range infos { + // info.BinaryPath is `//v/`; the schema + // PklProject lives at `//v/schema/pkl/PklProject`. + base := filepath.Dir(info.BinaryPath) + pklProject := filepath.Join(base, "schema", "pkl", "PklProject") + if _, err := os.Stat(pklProject); err != nil { + // Plugin binary exists but no PKL schema next to it (rare; + // some auth plugins, or a half-broken install). Skip — the + // CLI will fall back to remote URIs for this namespace. + continue + } + out[strings.ToLower(info.Name)] = pklProject + } + return out +} + +// Available returns plugins available for installation, filtered by f. +// Packages without "plugin" metadata are excluded. The channel resolves to +// DefaultChannel when f.Channel is empty so callers see only stable packages +// unless they opt in. +func (pm *PluginManager) Available(f AvailableFilter) ([]Plugin, error) { + orb, err := pm.clientFor(f.Channel) + if err != nil { + return nil, fmt.Errorf("building orbital client for channel: %w", err) + } + if err := orb.Refresh(); err != nil { + pm.logger.Warn("refresh failed, using cached repository data", "error", err) + } + + avail, err := orb.Available() + if err != nil { + return nil, fmt.Errorf("querying available packages: %w", err) + } + + plugins := make([]Plugin, 0, len(avail)) + for _, status := range avail { + if len(status.Available) == 0 { + continue + } + // Use the first plugin candidate as the descriptive package; if no + // candidate carries plugin metadata, skip this Status entirely. + var src *records.Package + for _, pkg := range status.Available { + if isPluginPackage(pkg) { + src = pkg + break + } + } + if src == nil { + continue + } + p := pluginFromPackage(src) + p.InstalledVersion = installedVersionOf(status.Available) + p.AvailableVersions = uniqueVersions(status.Available) + if matchesFilter(p, f) { + plugins = append(plugins, p) + } + } + + sort.Slice(plugins, func(i, j int) bool { + return plugins[i].Name < plugins[j].Name + }) + return plugins, nil +} + +// installedVersionOf returns the version short-string of the package marked +// Installed in pkgs, or "" if none is installed. +func installedVersionOf(pkgs []*records.Package) string { + for _, pkg := range pkgs { + if pkg != nil && pkg.Installed && pkg.Version != nil { + return versionString(pkg.Version) + } + } + return "" +} + +// Info returns detailed information about a single package, or nil if it is +// not found in any configured repository or doesn't carry plugin metadata. +// channel selects which channel to query; empty resolves to DefaultChannel. +// The orbital cache is refreshed before the lookup so callers don't see stale +// versions after a recent publish. +func (pm *PluginManager) Info(name, channel string) (*Plugin, error) { + orb, err := pm.clientFor(channel) + if err != nil { + return nil, fmt.Errorf("building orbital client for channel: %w", err) + } + if err := orb.Refresh(); err != nil { + pm.logger.Warn("refresh failed, using cached repository data", "error", err) + } + status, err := orb.AvailableFor(name) + if err != nil { + // orbital's AvailableFor returns "no available packages for: " + // when the package isn't in any of the requested channel's repos. + // That's a normal not-found, not an internal error. + if strings.Contains(err.Error(), "no available packages for") { + return nil, nil + } + return nil, fmt.Errorf("querying package info for %s: %w", name, err) + } + if status == nil || len(status.Available) == 0 { + return nil, nil + } + + var src *records.Package + for _, pkg := range status.Available { + if isPluginPackage(pkg) { + src = pkg + break + } + } + if src == nil { + return nil, nil + } + + p := pluginFromPackage(src) + p.InstalledVersion = installedVersionOf(status.Available) + p.AvailableVersions = uniqueVersions(status.Available) + return &p, nil +} + +// uniqueVersions returns the distinct version strings from pkgs in input +// order. Orbital can surface the same package twice when it lives both in +// the local tree (after install) and in a configured repo, so a naive +// append would produce duplicates like "0.1.0-dev.1, 0.1.0-dev.1". +func uniqueVersions(pkgs []*records.Package) []string { + seen := make(map[string]bool, len(pkgs)) + versions := make([]string, 0, len(pkgs)) + for _, pkg := range pkgs { + if pkg == nil || pkg.Header == nil || pkg.Version == nil { + continue + } + v := versionString(pkg.Version) + if seen[v] { + continue + } + seen[v] = true + versions = append(versions, v) + } + return versions +} + +// Install installs the requested packages via orbital. req.Channel selects +// which channel to install from; empty resolves to DefaultChannel. +func (pm *PluginManager) Install(req InstallRequest) (Response, error) { + orb, err := pm.clientFor(req.Channel) + if err != nil { + return Response{}, fmt.Errorf("building orbital client for channel: %w", err) + } + specs := packageSpecs(req.Packages) + if err := orb.Install(specs...); err != nil { + return Response{}, fmt.Errorf("installing packages: %w", err) + } + return pm.buildResponse(orb, req.Packages, "install") +} + +// Uninstall removes the requested packages via orbital. Uninstall operates on +// already-installed packages and is therefore channel-agnostic. +func (pm *PluginManager) Uninstall(req UninstallRequest) (Response, error) { + names := make([]string, len(req.Packages)) + for i, p := range req.Packages { + names[i] = p.Name + } + if err := pm.listOrb.Remove(names...); err != nil { + return Response{}, fmt.Errorf("removing packages: %w", err) + } + return pm.buildResponse(pm.listOrb, req.Packages, "remove") +} + +// Upgrade updates the requested packages via orbital. req.Channel selects +// which channel to upgrade against; empty resolves to DefaultChannel. +func (pm *PluginManager) Upgrade(req UpgradeRequest) (Response, error) { + orb, err := pm.clientFor(req.Channel) + if err != nil { + return Response{}, fmt.Errorf("building orbital client for channel: %w", err) + } + specs := packageSpecs(req.Packages) + if err := orb.Update(specs...); err != nil { + return Response{}, fmt.Errorf("upgrading packages: %w", err) + } + return pm.buildResponse(orb, req.Packages, "install") +} + +// buildResponse constructs a Response by querying the post-action state via +// the same orbital client that performed the action. +func (pm *PluginManager) buildResponse(orb orbitalClient, refs []PackageRef, action string) (Response, error) { + var resp Response + resp.RequiresRestart = true + for _, ref := range refs { + op := Operation{ + Name: ref.Name, + Version: ref.Version, + Action: action, + } + // Try to fill in type and resolved version from the repo metadata. + if status, err := orb.AvailableFor(ref.Name); err == nil && status != nil && len(status.Available) > 0 { + pkg := status.Available[0] + if pi, ok := pkg.Metadata["plugin"]; ok { + op.Type = pi["type"] + } + if op.Version == "" && pkg.Version != nil { + op.Version = versionString(pkg.Version) + } + } + resp.Operations = append(resp.Operations, op) + } + return resp, nil +} + +// packageSpecs converts PackageRefs to orbital spec strings ("name" or "name@version"). +func packageSpecs(refs []PackageRef) []string { + specs := make([]string, len(refs)) + for i, r := range refs { + if r.Version != "" { + specs[i] = r.Name + "@" + r.Version + } else { + specs[i] = r.Name + } + } + return specs +} + +// pluginFromPackage converts an orbital records.Package to a Plugin's +// descriptive fields. It deliberately does NOT populate InstalledVersion — +// that's a per-call concern: List sets it directly from the package (every +// package returned by orb.List() is installed by definition), while +// Available/Info derive it from records.Package.Installed via +// installedVersionOf so the field reflects orbital's actual state rather +// than the first available candidate. +func pluginFromPackage(pkg *records.Package) Plugin { + p := Plugin{ + Frozen: pkg.Frozen, + } + if pkg.Header == nil { + return p + } + p.Name = pkg.Name + p.Publisher = pkg.Publisher + p.License = pkg.License + p.Summary = pkg.Summary + p.Description = pkg.Description + p.Metadata = pkg.Metadata + if disp, ok := pkg.Metadata["display"]; ok { + p.Category = disp["category"] + p.Kind = disp["kind"] + } + if pi, ok := pkg.Metadata["plugin"]; ok { + p.Type = pi["type"] + p.Namespace = pi["namespace"] + } + return p +} + +// matchesFilter returns true if p satisfies every non-empty field in f. +func matchesFilter(p Plugin, f AvailableFilter) bool { + if f.Category != "" && p.Category != f.Category { + return false + } + if f.Type != "" && p.Type != f.Type { + return false + } + if f.Query != "" { + q := strings.ToLower(f.Query) + haystack := strings.ToLower(p.Name + " " + p.Summary + " " + p.Description) + if !strings.Contains(haystack, q) { + return false + } + } + return true +} diff --git a/internal/metastructure/plugin_manager/plugin_manager_test.go b/internal/metastructure/plugin_manager/plugin_manager_test.go new file mode 100644 index 00000000..bc4b31b6 --- /dev/null +++ b/internal/metastructure/plugin_manager/plugin_manager_test.go @@ -0,0 +1,787 @@ +//go:build unit + +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package plugin_manager + +import ( + "errors" + "log/slog" + "testing" + + "github.com/platform-engineering-labs/orbital/opm/records" + "github.com/platform-engineering-labs/orbital/ops" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// --------------------------------------------------------------------------- +// fake orbital client +// --------------------------------------------------------------------------- + +type fakeOrbitalClient struct { + refreshErr error + installErr error + removeErr error + updateErr error + availableForErr error + + installed []*records.Package + available map[string]*records.Status + + // capture args + installedSpecs []string + removedSpecs []string + updatedSpecs []string +} + +func (f *fakeOrbitalClient) Refresh() error { return f.refreshErr } +func (f *fakeOrbitalClient) Ready() bool { return true } + +func (f *fakeOrbitalClient) Install(packages ...string) error { + f.installedSpecs = packages + return f.installErr +} +func (f *fakeOrbitalClient) Remove(packages ...string) error { + f.removedSpecs = packages + return f.removeErr +} +func (f *fakeOrbitalClient) Update(packages ...string) error { + f.updatedSpecs = packages + return f.updateErr +} +func (f *fakeOrbitalClient) List() ([]*records.Package, error) { + return f.installed, nil +} +func (f *fakeOrbitalClient) Available() (map[string]*records.Status, error) { + return f.available, nil +} +func (f *fakeOrbitalClient) AvailableFor(name string) (*records.Status, error) { + if f.availableForErr != nil { + return nil, f.availableForErr + } + if f.available == nil { + return nil, nil + } + s, ok := f.available[name] + if !ok { + return nil, nil + } + return s, nil +} + +// --------------------------------------------------------------------------- +// helpers +// --------------------------------------------------------------------------- + +func newVersion(t *testing.T, s string) *ops.Version { + t.Helper() + v := &ops.Version{} + require.NoError(t, v.Parse(s)) + return v +} + +func newPackage(t *testing.T, name, version string, metadata map[string]map[string]string) *records.Package { + t.Helper() + return &records.Package{ + Header: &ops.Header{ + Name: name, + Version: newVersion(t, version), + Metadata: metadata, + }, + } +} + + +// --------------------------------------------------------------------------- +// constructor +// --------------------------------------------------------------------------- + +func TestNewPluginManager(t *testing.T) { + pm := newForTesting(slog.Default(), &fakeOrbitalClient{}) + require.NotNil(t, pm) + require.NotNil(t, pm.listOrb) + require.NotNil(t, pm.factory) +} + +// --------------------------------------------------------------------------- +// List +// --------------------------------------------------------------------------- + +func TestList_Empty(t *testing.T) { + pm := newForTesting(slog.Default(), &fakeOrbitalClient{}) + plugins, err := pm.List() + require.NoError(t, err) + assert.Empty(t, plugins) +} + +func TestList_ReturnsInstalledPlugins(t *testing.T) { + fake := &fakeOrbitalClient{ + installed: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.2.3", map[string]map[string]string{ + "plugin": {"type": "resource", "namespace": "aws"}, + "display": {"kind": "plugin", "category": "cloud"}, + }), + newPackage(t, "formae-plugin-tailscale", "0.5.0", map[string]map[string]string{ + "plugin": {"type": "network", "namespace": "tailscale"}, + "display": {"kind": "plugin"}, + }), + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.List() + require.NoError(t, err) + require.Len(t, plugins, 2) + + assert.Equal(t, "formae-plugin-aws", plugins[0].Name) + assert.Equal(t, "1.2.3", plugins[0].InstalledVersion) + assert.Equal(t, "resource", plugins[0].Type) + assert.Equal(t, "aws", plugins[0].Namespace) + assert.Equal(t, "cloud", plugins[0].Category) + + assert.Equal(t, "formae-plugin-tailscale", plugins[1].Name) + assert.Equal(t, "0.5.0", plugins[1].InstalledVersion) + assert.Equal(t, "network", plugins[1].Type) +} + +// --------------------------------------------------------------------------- +// List metadata filtering +// --------------------------------------------------------------------------- + +func TestList_IncludesMetapackages(t *testing.T) { + // Curated metapackages set display.kind == "metapackage" (and have no + // "plugin" metadata block since they have no runtime). They should + // surface in List the same way regular plugins do, with Kind set so + // the renderer can group them separately. + fake := &fakeOrbitalClient{ + installed: []*records.Package{ + newPackage(t, "standard", "0.1.0", map[string]map[string]string{ + "display": {"kind": "metapackage", "category": "bundle"}, + }), + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.List() + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "standard", plugins[0].Name) + assert.Equal(t, "metapackage", plugins[0].Kind) + assert.Equal(t, "bundle", plugins[0].Category) + assert.Equal(t, "", plugins[0].Type, "metapackages have no runtime type") +} + +func TestList_FiltersOutPackagesWithoutPluginMetadata(t *testing.T) { + // formae and pkl are binary tooling installed in the orbital tree but + // carry no "plugin" metadata; only the package with plugin metadata + // should surface through List. + fake := &fakeOrbitalClient{ + installed: []*records.Package{ + newPackage(t, "formae", "0.85.0", nil), + newPackage(t, "pkl", "0.31.0", nil), + newPackage(t, "formae-plugin-aws", "1.2.3", map[string]map[string]string{ + "plugin": {"type": "resource", "namespace": "aws"}, + "display": {"kind": "plugin"}, + }), + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.List() + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "formae-plugin-aws", plugins[0].Name) +} + +// --------------------------------------------------------------------------- +// Available +// --------------------------------------------------------------------------- + +func TestAvailable_NoFilter(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.2.3", map[string]map[string]string{ + "plugin": {"type": "resource", "namespace": "aws"}, + "display": {"kind": "plugin", "category": "cloud"}, + }), + newPackage(t, "formae-plugin-aws", "1.1.0", nil), + }, + }, + "formae-plugin-azure": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-azure", "0.1.0", map[string]map[string]string{ + "plugin": {"type": "resource", "namespace": "azure"}, + "display": {"kind": "plugin", "category": "cloud"}, + }), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{}) + require.NoError(t, err) + require.Len(t, plugins, 2) + + // Should be sorted by name. + assert.Equal(t, "formae-plugin-aws", plugins[0].Name) + assert.Equal(t, "formae-plugin-azure", plugins[1].Name) + + // AWS should have two available versions. + assert.Equal(t, []string{"1.2.3", "1.1.0"}, plugins[0].AvailableVersions) + // Azure should have one. + assert.Equal(t, []string{"0.1.0"}, plugins[1].AvailableVersions) +} + +func TestAvailable_FilterByCategory(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.0.0", map[string]map[string]string{ + "plugin": {"type": "resource"}, + "display": {"kind": "plugin", "category": "cloud"}, + }), + }, + }, + "formae-plugin-tailscale": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-tailscale", "0.3.0", map[string]map[string]string{ + "plugin": {"type": "network"}, + "display": {"kind": "plugin", "category": "networking"}, + }), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{Category: "networking"}) + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "formae-plugin-tailscale", plugins[0].Name) +} + +func TestAvailable_FilterByType(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.0.0", map[string]map[string]string{ + "plugin": {"type": "resource"}, + "display": {"kind": "plugin"}, + }), + }, + }, + "formae-plugin-tailscale": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-tailscale", "0.3.0", map[string]map[string]string{ + "plugin": {"type": "network"}, + "display": {"kind": "plugin"}, + }), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{Type: "resource"}) + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "formae-plugin-aws", plugins[0].Name) +} + +func TestAvailable_FilterByQuery(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.0.0", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}), + }, + }, + "formae-plugin-azure": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-azure", "0.1.0", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{Query: "azure"}) + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "formae-plugin-azure", plugins[0].Name) +} + +func TestAvailable_RefreshFailureDoesNotBlock(t *testing.T) { + fake := &fakeOrbitalClient{ + refreshErr: errors.New("network timeout"), + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.0.0", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{}) + require.NoError(t, err) + require.Len(t, plugins, 1) +} + +func TestAvailable_EmptyStatus(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-empty": {Available: nil}, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{}) + require.NoError(t, err) + assert.Empty(t, plugins) +} + +func TestAvailable_FiltersOutPackagesWithoutPluginMetadata(t *testing.T) { + // formae has no plugin metadata; sftp does. Only sftp surfaces. + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae": { + Available: []*records.Package{ + newPackage(t, "formae", "0.85.0", nil), + }, + }, + "formae-plugin-sftp": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-sftp", "0.1.0", map[string]map[string]string{ + "plugin": {"type": "resource", "namespace": "sftp"}, + "display": {"kind": "plugin"}, + }), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{}) + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "formae-plugin-sftp", plugins[0].Name) +} + +func TestAvailable_InstalledVersionFromInstalledFlag(t *testing.T) { + // Two candidates; v1.1.0 is the one orbital marks Installed. + // InstalledVersion must reflect that — NOT just Available[0].Version. + pkgInstalled := newPackage(t, "formae-plugin-aws", "1.1.0", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}) + pkgInstalled.Installed = true + pkgNewer := newPackage(t, "formae-plugin-aws", "1.2.0", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}) + + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{pkgNewer, pkgInstalled}, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{}) + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "1.1.0", plugins[0].InstalledVersion) +} + +func TestAvailable_NoInstalledVersionWhenNoneInstalled(t *testing.T) { + // Nothing in Available has Installed=true → InstalledVersion is empty. + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-sftp": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-sftp", "0.1.0", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + plugins, err := pm.Available(AvailableFilter{}) + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "", plugins[0].InstalledVersion) +} + +// --------------------------------------------------------------------------- +// Channel routing +// --------------------------------------------------------------------------- + +// channelFakes builds a factory that hands out a different fake per channel. +func channelFakes(byChannel map[string]*fakeOrbitalClient) orbitalFactory { + return func(channel string) (orbitalClient, error) { + if c, ok := byChannel[channel]; ok { + return c, nil + } + return &fakeOrbitalClient{}, nil + } +} + +func TestAvailable_DefaultsToStableChannel(t *testing.T) { + stable := &fakeOrbitalClient{available: map[string]*records.Status{}} + dev := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-sftp": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-sftp", "0.1.0-dev.1", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}), + }, + }, + }, + } + pm := newForTestingWithFactory(slog.Default(), &fakeOrbitalClient{}, + channelFakes(map[string]*fakeOrbitalClient{ + "stable": stable, + "dev": dev, + })) + + plugins, err := pm.Available(AvailableFilter{}) + require.NoError(t, err) + assert.Empty(t, plugins, "default search must hit stable, which is empty") +} + +func TestAvailable_ChannelOverrideReturnsDevPlugin(t *testing.T) { + stable := &fakeOrbitalClient{available: map[string]*records.Status{}} + dev := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-sftp": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-sftp", "0.1.0-dev.1", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}), + }, + }, + }, + } + pm := newForTestingWithFactory(slog.Default(), &fakeOrbitalClient{}, + channelFakes(map[string]*fakeOrbitalClient{ + "stable": stable, + "dev": dev, + })) + + plugins, err := pm.Available(AvailableFilter{Channel: "dev"}) + require.NoError(t, err) + require.Len(t, plugins, 1) + assert.Equal(t, "formae-plugin-sftp", plugins[0].Name) +} + +func TestInfo_ChannelOverride(t *testing.T) { + stable := &fakeOrbitalClient{available: map[string]*records.Status{}} + dev := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-sftp": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-sftp", "0.1.0-dev.1", + map[string]map[string]string{"plugin": {"type": "resource"}, "display": {"kind": "plugin"}}), + }, + }, + }, + } + pm := newForTestingWithFactory(slog.Default(), &fakeOrbitalClient{}, + channelFakes(map[string]*fakeOrbitalClient{ + "stable": stable, + "dev": dev, + })) + + pStable, err := pm.Info("formae-plugin-sftp", "") + require.NoError(t, err) + assert.Nil(t, pStable, "info on default stable channel should not find dev-only plugin") + + pDev, err := pm.Info("formae-plugin-sftp", "dev") + require.NoError(t, err) + require.NotNil(t, pDev) + assert.Equal(t, "formae-plugin-sftp", pDev.Name) +} + +func TestInstall_ChannelRoutesToCorrectClient(t *testing.T) { + stable := &fakeOrbitalClient{} + dev := &fakeOrbitalClient{} + pm := newForTestingWithFactory(slog.Default(), &fakeOrbitalClient{}, + channelFakes(map[string]*fakeOrbitalClient{ + "stable": stable, + "dev": dev, + })) + + _, err := pm.Install(InstallRequest{ + Packages: []PackageRef{{Name: "formae-plugin-sftp", Version: "0.1.0-dev.1"}}, + Channel: "dev", + }) + require.NoError(t, err) + assert.Equal(t, []string{"formae-plugin-sftp@0.1.0-dev.1"}, dev.installedSpecs, + "install with --channel dev must route to the dev client") + assert.Empty(t, stable.installedSpecs, "stable client should not see this install") +} + +// --------------------------------------------------------------------------- +// Info +// --------------------------------------------------------------------------- + +func TestInfo_Found(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.2.0", map[string]map[string]string{ + "plugin": {"type": "resource", "namespace": "aws"}, + "display": {"kind": "plugin"}, + }), + newPackage(t, "formae-plugin-aws", "1.1.0", nil), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + p, err := pm.Info("formae-plugin-aws", "") + require.NoError(t, err) + require.NotNil(t, p) + assert.Equal(t, "formae-plugin-aws", p.Name) + assert.Equal(t, "resource", p.Type) + assert.Equal(t, []string{"1.2.0", "1.1.0"}, p.AvailableVersions) +} + +func TestInfo_NotFound(t *testing.T) { + fake := &fakeOrbitalClient{available: map[string]*records.Status{}} + pm := newForTesting(slog.Default(), fake) + + p, err := pm.Info("nonexistent", "") + require.NoError(t, err) + assert.Nil(t, p) +} + +// TestInfo_NotFoundFromOrbitalError verifies that orbital's "no available +// packages for: X" error (returned when X is in none of the configured +// channel's repos) is translated to a not-found nil result rather than +// bubbling up as a 500. +func TestInfo_NotFoundFromOrbitalError(t *testing.T) { + fake := &fakeOrbitalClient{ + availableForErr: errors.New("no available packages for: ghost"), + } + pm := newForTesting(slog.Default(), fake) + + p, err := pm.Info("ghost", "dev") + require.NoError(t, err) + assert.Nil(t, p) +} + +func TestInfo_NotFoundWhenNoPluginMetadata(t *testing.T) { + // `formae` has no "plugin" metadata; Info treats it as not-a-plugin. + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae": { + Available: []*records.Package{ + newPackage(t, "formae", "0.85.0", nil), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + p, err := pm.Info("formae", "") + require.NoError(t, err) + assert.Nil(t, p) +} + +// --------------------------------------------------------------------------- +// Install +// --------------------------------------------------------------------------- + +func TestInstall_Success(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.2.0", map[string]map[string]string{ + "plugin": {"type": "resource"}, + "display": {"kind": "plugin"}, + }), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + resp, err := pm.Install(InstallRequest{ + Packages: []PackageRef{ + {Name: "formae-plugin-aws", Version: "1.2.0"}, + }, + }) + require.NoError(t, err) + require.Len(t, resp.Operations, 1) + assert.Equal(t, "formae-plugin-aws", resp.Operations[0].Name) + assert.Equal(t, "install", resp.Operations[0].Action) + assert.Equal(t, "1.2.0", resp.Operations[0].Version) + assert.Equal(t, "resource", resp.Operations[0].Type) + assert.True(t, resp.RequiresRestart) + + // Verify the spec passed to orbital. + assert.Equal(t, []string{"formae-plugin-aws@1.2.0"}, fake.installedSpecs) +} + +func TestInstall_NoVersion(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "2.0.0", nil), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + resp, err := pm.Install(InstallRequest{ + Packages: []PackageRef{{Name: "formae-plugin-aws"}}, + }) + require.NoError(t, err) + assert.Equal(t, []string{"formae-plugin-aws"}, fake.installedSpecs) + assert.Equal(t, "2.0.0", resp.Operations[0].Version) +} + +func TestInstall_Error(t *testing.T) { + fake := &fakeOrbitalClient{installErr: errors.New("conflict")} + pm := newForTesting(slog.Default(), fake) + + _, err := pm.Install(InstallRequest{ + Packages: []PackageRef{{Name: "bad"}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "installing packages") +} + +// --------------------------------------------------------------------------- +// Uninstall +// --------------------------------------------------------------------------- + +func TestUninstall_Success(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "1.0.0", map[string]map[string]string{ + "plugin": {"type": "resource"}, + "display": {"kind": "plugin"}, + }), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + resp, err := pm.Uninstall(UninstallRequest{ + Packages: []PackageRef{{Name: "formae-plugin-aws"}}, + }) + require.NoError(t, err) + require.Len(t, resp.Operations, 1) + assert.Equal(t, "remove", resp.Operations[0].Action) + assert.Equal(t, []string{"formae-plugin-aws"}, fake.removedSpecs) +} + +func TestUninstall_Error(t *testing.T) { + fake := &fakeOrbitalClient{removeErr: errors.New("in use")} + pm := newForTesting(slog.Default(), fake) + + _, err := pm.Uninstall(UninstallRequest{ + Packages: []PackageRef{{Name: "locked"}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "removing packages") +} + +// --------------------------------------------------------------------------- +// Upgrade +// --------------------------------------------------------------------------- + +func TestUpgrade_Success(t *testing.T) { + fake := &fakeOrbitalClient{ + available: map[string]*records.Status{ + "formae-plugin-aws": { + Available: []*records.Package{ + newPackage(t, "formae-plugin-aws", "2.0.0", map[string]map[string]string{ + "plugin": {"type": "resource"}, + "display": {"kind": "plugin"}, + }), + }, + }, + }, + } + pm := newForTesting(slog.Default(), fake) + + resp, err := pm.Upgrade(UpgradeRequest{ + Packages: []PackageRef{{Name: "formae-plugin-aws", Version: "2.0.0"}}, + }) + require.NoError(t, err) + require.Len(t, resp.Operations, 1) + assert.Equal(t, "install", resp.Operations[0].Action) + assert.Equal(t, "2.0.0", resp.Operations[0].Version) + assert.Equal(t, []string{"formae-plugin-aws@2.0.0"}, fake.updatedSpecs) +} + +func TestUpgrade_Error(t *testing.T) { + fake := &fakeOrbitalClient{updateErr: errors.New("no update")} + pm := newForTesting(slog.Default(), fake) + + _, err := pm.Upgrade(UpgradeRequest{ + Packages: []PackageRef{{Name: "x"}}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "upgrading packages") +} + +// --------------------------------------------------------------------------- +// pluginFromPackage +// --------------------------------------------------------------------------- + +func TestPluginFromPackage_NilHeader(t *testing.T) { + pkg := &records.Package{Header: nil} + p := pluginFromPackage(pkg) + assert.Equal(t, "", p.Name) + assert.Equal(t, "", p.InstalledVersion) +} + +func TestPluginFromPackage_NilVersion(t *testing.T) { + pkg := &records.Package{ + Header: &ops.Header{ + Name: "test", + Version: nil, + }, + } + p := pluginFromPackage(pkg) + assert.Equal(t, "test", p.Name) + assert.Equal(t, "", p.InstalledVersion) +} + +// --------------------------------------------------------------------------- +// matchesFilter +// --------------------------------------------------------------------------- + +func TestMatchesFilter_AllEmpty(t *testing.T) { + assert.True(t, matchesFilter(Plugin{Name: "x"}, AvailableFilter{})) +} + +func TestMatchesFilter_QueryCaseInsensitive(t *testing.T) { + p := Plugin{Name: "formae-plugin-AWS", Summary: "Amazon Web Services"} + assert.True(t, matchesFilter(p, AvailableFilter{Query: "amazon"})) + assert.True(t, matchesFilter(p, AvailableFilter{Query: "AWS"})) + assert.False(t, matchesFilter(p, AvailableFilter{Query: "gcp"})) +} diff --git a/internal/opsmgr/opsmgr.go b/internal/opsmgr/opsmgr.go index 653acf9e..cb9d4a64 100644 --- a/internal/opsmgr/opsmgr.go +++ b/internal/opsmgr/opsmgr.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" + pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" "github.com/platform-engineering-labs/orbital/mgr" "github.com/platform-engineering-labs/orbital/opm/security" "github.com/platform-engineering-labs/orbital/opm/tree" @@ -18,26 +19,64 @@ import ( "github.com/platform-engineering-labs/orbital/platform" ) +// New constructs a Manager from a single URL + channel. +// Kept for backwards compatibility. Prefer NewFromRepositories. func New(logger *slog.Logger, uri url.URL, channel string) (*mgr.Manager, error) { + repos := []pkgmodel.Repository{{URI: uri, Type: pkgmodel.RepositoryTypeBinary}} + return NewFromRepositories(logger, repos, channel) +} + +// NewFromRepositories constructs a Manager with all configured repos loaded. +// Channel (if non-empty) overrides the URI fragment for every repo. +func NewFromRepositories(logger *slog.Logger, repos []pkgmodel.Repository, channel string) (*mgr.Manager, error) { + return newManager(logger, repos, channel) +} + +// NewFromRepositoriesFiltered constructs a Manager containing only repos whose +// type is in the allowed set. Empty allowed = all types. +func NewFromRepositoriesFiltered(logger *slog.Logger, repos []pkgmodel.Repository, channel string, allowed ...pkgmodel.RepositoryType) (*mgr.Manager, error) { + if len(allowed) == 0 { + return newManager(logger, repos, channel) + } + allow := make(map[pkgmodel.RepositoryType]bool, len(allowed)) + for _, a := range allowed { + allow[a] = true + } + filtered := make([]pkgmodel.Repository, 0, len(repos)) + for _, r := range repos { + if allow[r.Type] { + filtered = append(filtered, r) + } + } + if len(filtered) == 0 { + return nil, fmt.Errorf("no matching repositories after filtering") + } + return newManager(logger, filtered, channel) +} + +func newManager(logger *slog.Logger, repos []pkgmodel.Repository, channel string) (*mgr.Manager, error) { binPath, err := os.Executable() if err != nil { return nil, fmt.Errorf("could not determine binary path: %w", err) } - if channel != "" { - uri.Fragment = channel + opsRepos := make([]ops.Repository, 0, len(repos)) + for _, r := range repos { + uri := r.URI + if channel != "" { + uri.Fragment = channel + } + opsRepos = append(opsRepos, ops.Repository{Uri: uri, Priority: 0, Enabled: true}) + } + + if len(opsRepos) == 0 { + return nil, fmt.Errorf("no repositories configured") } return mgr.New(logger, filepath.Dir(filepath.Dir(binPath)), &tree.Config{ - OS: platform.Current().OS, - Arch: platform.Current().Arch, - Security: security.Default, - Repositories: []ops.Repository{ - { - Uri: uri, - Priority: 0, - Enabled: true, - }, - }, + OS: platform.Current().OS, + Arch: platform.Current().Arch, + Security: security.Default, + Repositories: opsRepos, }) } diff --git a/internal/schema/json/json.go b/internal/schema/json/json.go index 9532f98a..4443fac5 100644 --- a/internal/schema/json/json.go +++ b/internal/schema/json/json.go @@ -127,7 +127,7 @@ func highlight(code []byte) ([]byte, error) { return buf.Bytes(), nil } -func (j JSON) GenerateSourceCode(forma *model.Forma, targetPath string, includes []string, schemaLocation schema.SchemaLocation) (schema.GenerateSourcesResult, error) { +func (j JSON) GenerateSourceCode(forma *model.Forma, targetPath string, includes []string, options *schema.SerializeOptions) (schema.GenerateSourcesResult, error) { return schema.GenerateSourcesResult{}, errors.ErrUnsupported } diff --git a/internal/schema/pkl/assets/formae/Config.pkl b/internal/schema/pkl/assets/formae/Config.pkl index bc2fa433..312eb512 100644 --- a/internal/schema/pkl/assets/formae/Config.pkl +++ b/internal/schema/pkl/assets/formae/Config.pkl @@ -236,12 +236,36 @@ class ApiConfig { port: Port = 49684 } +class Repository { + /// Orbital repository URI (optionally with #channel fragment). + uri: Uri + + /// Discriminator: "binary" for the formae binary / tooling repo; + /// "formae-plugin" for a plugin repo consulted by `formae plugin *`. + type: "binary"|"formae-plugin" +} + class ArtifactConfig { + @Deprecated { message = "Use artifacts.repositories instead" } url: Uri = "https://hub.platform.engineering/repos/platform.engineering/pel#stable" + @Deprecated { message = "Use repository-specific credentials" } username: String? + @Deprecated { message = "Use repository-specific credentials" } password: String? + + /// List of orbital repositories to consult for binary and plugin packages. + repositories: Listing = new { + new { + uri = "https://hub.platform.engineering/repos/platform.engineering/pel#stable" + type = "binary" + } + new { + uri = "https://hub.platform.engineering/repos/platform.engineering/community#stable" + type = "formae-plugin" + } + } } class CliConfig { diff --git a/internal/schema/pkl/model/config.go b/internal/schema/pkl/model/config.go index 7645e243..43b2f49a 100644 --- a/internal/schema/pkl/model/config.go +++ b/internal/schema/pkl/model/config.go @@ -17,6 +17,7 @@ func init() { pkl.RegisterMapping("formae.Config#LabelConfig", LabelConfig{}) pkl.RegisterMapping("formae.Config#MatchFilter", MatchFilter{}) pkl.RegisterMapping("formae.Config#FilterCondition", FilterCondition{}) + pkl.RegisterMapping("formae.Config#Repository", Repository{}) } // ResourcePlugin nested types used when decoding BaseResourcePluginConfig @@ -174,10 +175,16 @@ type APIConfig struct { Port int32 `pkl:"port"` } +type Repository struct { + URI url.URL `pkl:"uri"` + Type string `pkl:"type"` +} + type ArtifactConfig struct { - URL url.URL `pkl:"url"` - Username string `pkl:"username"` - Password string `pkl:"password"` + URL url.URL `pkl:"url"` + Username string `pkl:"username"` + Password string `pkl:"password"` + Repositories []*Repository `pkl:"repositories"` } type CliConfig struct { diff --git a/internal/schema/pkl/package_resolver.go b/internal/schema/pkl/package_resolver.go index d6e0fcbb..61eaefcb 100644 --- a/internal/schema/pkl/package_resolver.go +++ b/internal/schema/pkl/package_resolver.go @@ -323,13 +323,3 @@ func (r *PackageResolver) GetPackages() []Package { func (r *PackageResolver) IsUsingLocalSchemas() bool { return r.useLocalSchemas } - -// HasRemotePackages returns true if any packages are remote (need pkl project resolve) -func (r *PackageResolver) HasRemotePackages() bool { - for _, pkg := range r.packages { - if !pkg.IsLocal { - return true - } - } - return false -} diff --git a/internal/schema/pkl/pkl.go b/internal/schema/pkl/pkl.go index be6b1c79..59c2cfbb 100644 --- a/internal/schema/pkl/pkl.go +++ b/internal/schema/pkl/pkl.go @@ -21,7 +21,6 @@ import ( "time" pklgo "github.com/apple/pkl-go/pkl" - "github.com/platform-engineering-labs/formae" "github.com/platform-engineering-labs/formae/internal/schema" pklmodel "github.com/platform-engineering-labs/formae/internal/schema/pkl/model" pkgmodel "github.com/platform-engineering-labs/formae/pkg/model" @@ -224,11 +223,7 @@ func translateConfig(config *pklmodel.Config) *pkgmodel.Config { Auth: translateAuthConfig(&config.Agent.Auth), ResourcePlugins: translateResourcePluginConfigs(config.Agent.ResourcePlugins), }, - Artifacts: pkgmodel.ArtifactConfig{ - URL: config.Artifacts.URL, - Username: config.Artifacts.Username, - Password: config.Artifacts.Password, - }, + Artifacts: translateArtifactConfig(&config.Artifacts), Cli: pkgmodel.CliConfig{ API: pkgmodel.APIConfig{ URL: config.Cli.API.URL, @@ -248,9 +243,38 @@ func translateConfig(config *pklmodel.Config) *pkgmodel.Config { // Warn when global settings conflict with per-plugin overrides checkResourcePluginDeprecations(&translated) + // Synthesize Repositories from legacy flat fields and emit deprecation warnings + emitArtifactDeprecationWarnings(&translated) + return &translated } +// emitArtifactDeprecationWarnings synthesizes a canonical Repositories entry from +// the legacy flat URL field and appends deprecation warnings to translated.Warnings. +func emitArtifactDeprecationWarnings(translated *pkgmodel.Config) { + a := &translated.Artifacts + // When the user config uses the legacy flat fields and hasn't migrated + // to repositories, synthesize a single binary repository entry and warn. + if a.URL.String() != "" && len(a.Repositories) == 0 { + a.Repositories = []pkgmodel.Repository{ + {URI: a.URL, Type: pkgmodel.RepositoryTypeBinary}, + } + w := "artifacts.url is deprecated; migrate to artifacts.repositories. The URL has been loaded as a 'binary' repository for this release." + slog.Warn(w) + translated.Warnings = append(translated.Warnings, w) + } + if a.Username != "" { + w := "artifacts.username is deprecated; repository credentials will be per-repo in a future release" + slog.Warn(w) + translated.Warnings = append(translated.Warnings, w) + } + if a.Password != "" { + w := "artifacts.password is deprecated; repository credentials will be per-repo in a future release" + slog.Warn(w) + translated.Warnings = append(translated.Warnings, w) + } +} + // checkResourcePluginDeprecations warns when global settings conflict with per-plugin overrides. func checkResourcePluginDeprecations(translated *pkgmodel.Config) { if len(translated.Agent.ResourcePlugins) == 0 { @@ -422,67 +446,67 @@ func (p PKL) Evaluate(path string, cmd pkgmodel.Command, mode pkgmodel.FormaAppl return forma, nil } -func (p PKL) GenerateSourceCode(forma *pkgmodel.Forma, path string, includes []string, schemaLocation schema.SchemaLocation) (schema.GenerateSourcesResult, error) { +func (p PKL) GenerateSourceCode(forma *pkgmodel.Forma, path string, includes []string, options *schema.SerializeOptions) (schema.GenerateSourcesResult, error) { res := schema.GenerateSourcesResult{} - code, err := p.SerializeForma(forma, &schema.SerializeOptions{Schema: "pkl", SchemaLocation: schemaLocation}) - if err != nil { - slog.Error(err.Error()) - return schema.GenerateSourcesResult{}, schema.ErrFailedToGenerateSources + if options == nil { + options = &schema.SerializeOptions{Schema: "pkl"} + } + if options.Schema == "" { + options.Schema = "pkl" + } + schemaLocation := options.SchemaLocation + if schemaLocation == "" { + schemaLocation = schema.SchemaLocationRemote } - res.ResourceCount = len(forma.Resources) - // add .pkl to path if not present if !strings.HasSuffix(path, ".pkl") { path = path + ".pkl" } res.TargetPath = path - // Ensure parent directory exists parentDir := filepath.Dir(path) - if err = os.MkdirAll(parentDir, 0755); err != nil { + if err := os.MkdirAll(parentDir, 0755); err != nil { return schema.GenerateSourcesResult{}, fmt.Errorf("failed to create parent directory: %v", err) } projectFile := filepath.Join(parentDir, "PklProject") - if _, err = os.Stat(projectFile); os.IsNotExist(err) { - // Build package dependencies using PackageResolver - resolver := NewPackageResolver() - - // Configure local schema resolution if requested - if schemaLocation == schema.SchemaLocationLocal { - homeDir, err := os.UserHomeDir() - if err == nil { - pluginsDir := filepath.Join(homeDir, ".pel", "formae", "plugins") - resolver.WithLocalSchemas(pluginsDir) - } - } - - resolver.Add("formae", "pkl", formae.Version) - - // Extract namespaces from forma resources - for _, res := range forma.Resources { - ns := strings.ToLower(res.Namespace()) - resolver.Add(ns, ns, resolver.InstalledVersion(ns)) - } - - // No PklProject exists, initialize it with resolved packages - err = p.ProjectInit(parentDir, resolver.GetPackageStrings(), schemaLocation) - if err != nil { + if _, err := os.Stat(projectFile); err == nil { + // Case 1: target dir has an existing PklProject — reuse its deps so the + // generated .pkl resolves cleanly when the user later evaluates it under + // their own project. + deps, parseErr := parsePklProjectDeps(projectFile) + if parseErr != nil { + return schema.GenerateSourcesResult{}, fmt.Errorf("failed to parse existing PklProject %q: %w", projectFile, parseErr) + } + options.Dependencies = deps + } else if os.IsNotExist(err) { + // Case 2: no existing PklProject — discover deps from options.LocalPluginDir + // and pin them so the generator and target ProjectInit use identical specs. + deps := resolveIncludes(forma, options) + options.Dependencies = deps + + if err := p.ProjectInit(parentDir, deps, schemaLocation); err != nil { return schema.GenerateSourcesResult{}, fmt.Errorf("failed to initialize Pkl project: %v", err) } res.InitializedNewProject = true res.ProjectPath = parentDir fmt.Println("Initialized new Pkl project at", parentDir) + } else { + return schema.GenerateSourcesResult{}, fmt.Errorf("failed to stat %q: %w", projectFile, err) } - err = os.WriteFile(path, []byte(code), 0644) + code, err := p.SerializeForma(forma, options) if err != nil { + slog.Error(err.Error()) + return schema.GenerateSourcesResult{}, schema.ErrFailedToGenerateSources + } + res.ResourceCount = len(forma.Resources) + + if err := os.WriteFile(path, []byte(code), 0644); err != nil { return schema.GenerateSourcesResult{}, fmt.Errorf("failed to write Pkl file: %v", err) } - // Check if PklProject.deps.json exists after writing the Pkl file - // Only warn for remote schema location since local schemas don't need resolution depsFile := filepath.Join(parentDir, "PklProject.deps.json") if _, err := os.Stat(depsFile); os.IsNotExist(err) && schemaLocation == schema.SchemaLocationRemote { res.Warnings = append(res.Warnings, fmt.Sprintf("Pkl dependencies not resolved. Run 'pkl project resolve' in '%s' to resolve Pkl dependencies.", parentDir)) @@ -686,6 +710,21 @@ func translateRetryConfig(rc *pklmodel.RetryConfig) pkgmodel.RetryConfig { } } +func translateArtifactConfig(ac *pklmodel.ArtifactConfig) pkgmodel.ArtifactConfig { + result := pkgmodel.ArtifactConfig{ + URL: ac.URL, + Username: ac.Username, + Password: ac.Password, + } + for _, r := range ac.Repositories { + result.Repositories = append(result.Repositories, pkgmodel.Repository{ + URI: r.URI, + Type: pkgmodel.RepositoryType(r.Type), + }) + } + return result +} + func translateResourcePluginConfigs(objects []pklgo.Object) []pkgmodel.ResourcePluginUserConfig { if len(objects) == 0 { return nil diff --git a/internal/schema/pkl/pkl_serialize.go b/internal/schema/pkl/pkl_serialize.go index f361687e..91d8d838 100644 --- a/internal/schema/pkl/pkl_serialize.go +++ b/internal/schema/pkl/pkl_serialize.go @@ -37,32 +37,12 @@ func (p PKL) serializeWithPKL(data *model.Forma, options *schema.SerializeOption } defer func() { _ = os.RemoveAll(tempDir) }() - // Build package dependencies using PackageResolver - resolver := NewPackageResolver() + includes := resolveIncludes(data, options) - // Configure local schema resolution if requested schemaLocation := schema.SchemaLocationRemote if options != nil && options.SchemaLocation != "" { schemaLocation = options.SchemaLocation } - if schemaLocation == schema.SchemaLocationLocal { - homeDir, err := os.UserHomeDir() - if err == nil { - pluginsDir := filepath.Join(homeDir, ".pel", "formae", "plugins") - resolver.WithLocalSchemas(pluginsDir) - } - } - - resolver.Add("formae", "pkl", formae.Version) - - // Extract namespaces from the data and add them as dependencies - namespaces := extractNamespaces(data) - for ns := range namespaces { - // Each namespace uses itself as the plugin name (e.g., aws uses aws plugin) - resolver.Add(ns, ns, resolver.InstalledVersion(ns)) - } - - includes := resolver.GetPackageStrings() err = fs.WalkDir(generator, ".", func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -141,6 +121,32 @@ func (p PKL) serializeWithPKL(data *model.Forma, options *schema.SerializeOption return Format(textOutput), nil } +// resolveIncludes returns the package specs to use for the PKL generator's temp +// PklProject. Caller-supplied options.Dependencies take priority; otherwise we +// build the spec list from options.LocalPluginDir (when SchemaLocation == Local +// and dir is non-empty) or fall back to remote-only. +func resolveIncludes(data *model.Forma, options *schema.SerializeOptions) []string { + if options != nil && len(options.Dependencies) > 0 { + return options.Dependencies + } + + resolver := NewPackageResolver() + + schemaLocation := schema.SchemaLocationRemote + if options != nil && options.SchemaLocation != "" { + schemaLocation = options.SchemaLocation + } + if schemaLocation == schema.SchemaLocationLocal && options != nil && options.LocalPluginDir != "" { + resolver.WithLocalSchemas(options.LocalPluginDir) + } + + resolver.Add("formae", "pkl", formae.Version) + for ns := range extractNamespaces(data) { + resolver.Add(ns, ns, resolver.InstalledVersion(ns)) + } + return resolver.GetPackageStrings() +} + // extractNamespaces extracts unique namespaces from the data. // It handles both *model.Resource and *model.Forma types. func extractNamespaces(data any) map[string]struct{} { diff --git a/internal/schema/pkl/pkl_serialize_test.go b/internal/schema/pkl/pkl_serialize_test.go new file mode 100644 index 00000000..a4b9b53d --- /dev/null +++ b/internal/schema/pkl/pkl_serialize_test.go @@ -0,0 +1,50 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package pkl + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/platform-engineering-labs/formae/internal/schema" + "github.com/platform-engineering-labs/formae/pkg/model" +) + +func TestResolveIncludes_PreResolvedDepsTakePrecedence(t *testing.T) { + forma := &model.Forma{Resources: []model.Resource{{Type: "AWS::S3::Bucket"}}} + options := &schema.SerializeOptions{ + Schema: "pkl", + SchemaLocation: schema.SchemaLocationLocal, + LocalPluginDir: "/this/path/should/not/be/touched", + Dependencies: []string{ + "pkl.formae@0.85.0", + "local:aws:/some/path/PklProject", + }, + } + + got := resolveIncludes(forma, options) + + assert.ElementsMatch(t, []string{ + "pkl.formae@0.85.0", + "local:aws:/some/path/PklProject", + }, got) +} + +func TestResolveIncludes_RemoteOnlyWhenNoDirAndNoDeps(t *testing.T) { + forma := &model.Forma{Resources: []model.Resource{{Type: "AWS::S3::Bucket"}}} + options := &schema.SerializeOptions{ + Schema: "pkl", + SchemaLocation: schema.SchemaLocationRemote, + } + + got := resolveIncludes(forma, options) + + // formae version is the binary's compile-time version (test build = "0.0.0"), + // so it's filtered out of the includes by the resolver. We expect only the + // remote aws entry — and since aws version is "" (no installed version), it's + // added as a plain namespace. + assert.Contains(t, got, "aws.aws@") +} diff --git a/internal/schema/pkl/pkl_test.go b/internal/schema/pkl/pkl_test.go index adb026d1..cfc26ad3 100644 --- a/internal/schema/pkl/pkl_test.go +++ b/internal/schema/pkl/pkl_test.go @@ -215,3 +215,23 @@ func TestDeprecationWarning_GlobalRetryWithPerPlugin(t *testing.T) { assert.Contains(t, config.Warnings[0], "agent.retry") assert.Contains(t, config.Warnings[0], "deprecated") } + +func TestDeprecationWarning_LegacyArtifactsURL(t *testing.T) { + p := PKL{} + config, err := p.FormaeConfig("./testdata/config/test_deprecation_artifacts.pkl") + require.NoError(t, err) + + // Expect three warnings: url, username, password + require.Len(t, config.Warnings, 3) + assert.Contains(t, config.Warnings[0], "artifacts.url") + assert.Contains(t, config.Warnings[0], "deprecated") + assert.Contains(t, config.Warnings[1], "artifacts.username") + assert.Contains(t, config.Warnings[1], "deprecated") + assert.Contains(t, config.Warnings[2], "artifacts.password") + assert.Contains(t, config.Warnings[2], "deprecated") + + // The URL should have been synthesized into Repositories as a binary entry + require.Len(t, config.Artifacts.Repositories, 1) + assert.Equal(t, "example.org", config.Artifacts.Repositories[0].URI.Host) + assert.Equal(t, model.RepositoryTypeBinary, config.Artifacts.Repositories[0].Type) +} diff --git a/internal/schema/pkl/project_deps.go b/internal/schema/pkl/project_deps.go new file mode 100644 index 00000000..a4224077 --- /dev/null +++ b/internal/schema/pkl/project_deps.go @@ -0,0 +1,110 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package pkl + +import ( + "fmt" + "os" + "regexp" + "strings" +) + +// remoteURIPattern matches the URI shape emitted by PklProjectTemplate.pkl: +// package:///plugins//schema/pkl//@ +var remoteURIPattern = regexp.MustCompile(`^package://[^/]+/plugins/([^/]+)/schema/pkl/[^/]+/([^@]+)@(.+)$`) + +// importPattern matches: [""] = import("") +var importPattern = regexp.MustCompile(`\[\s*"([^"]+)"\s*\]\s*=\s*import\(\s*"([^"]+)"\s*\)`) + +// uriPattern matches the inner uri assignment of a remote dep: +// uri = "package://..." +var uriPattern = regexp.MustCompile(`uri\s*=\s*"([^"]+)"`) + +// nameKeyPattern matches the start of a remote dep block: [""] { +var nameKeyPattern = regexp.MustCompile(`\[\s*"([^"]+)"\s*\]\s*\{`) + +// parsePklProjectDeps reads the `dependencies { ... }` block from an existing +// PklProject file and returns the entries as package specs in the format +// emitted by PackageResolver: +// - remote: ".@" +// - local: "local::" +// +// Returns an error if the file cannot be read. +func parsePklProjectDeps(pklProjectPath string) ([]string, error) { + data, err := os.ReadFile(pklProjectPath) + if err != nil { + return nil, fmt.Errorf("read PklProject %q: %w", pklProjectPath, err) + } + + content := string(data) + + // Find the dependencies { ... } block. We don't try to be a full Pkl parser — + // we just scan from "dependencies {" to its matching "}". + depsStart := strings.Index(content, "dependencies") + if depsStart < 0 { + return nil, nil + } + openBrace := strings.Index(content[depsStart:], "{") + if openBrace < 0 { + return nil, fmt.Errorf("malformed dependencies block in %q", pklProjectPath) + } + depsBody, ok := scanBalancedBraces(content[depsStart+openBrace:]) + if !ok { + return nil, fmt.Errorf("unbalanced dependencies block in %q", pklProjectPath) + } + + var out []string + + // Local deps: [""] = import("") + for _, m := range importPattern.FindAllStringSubmatch(depsBody, -1) { + out = append(out, fmt.Sprintf("local:%s:%s", m[1], m[2])) + } + + // Remote deps: [""] { uri = "package://..." } + // Walk each [""] { block and look for the uri inside it. + starts := nameKeyPattern.FindAllStringSubmatchIndex(depsBody, -1) + for _, s := range starts { + // Block body starts after the opening brace at index s[1]-1. + blockBody, ok := scanBalancedBraces(depsBody[s[1]-1:]) + if !ok { + continue + } + uriMatch := uriPattern.FindStringSubmatch(blockBody) + if len(uriMatch) < 2 { + continue + } + uriParts := remoteURIPattern.FindStringSubmatch(uriMatch[1]) + if len(uriParts) != 4 { + continue + } + plugin, pkgName, version := uriParts[1], uriParts[2], uriParts[3] + out = append(out, fmt.Sprintf("%s.%s@%s", plugin, pkgName, version)) + } + + return out, nil +} + +// scanBalancedBraces returns the substring inside the first balanced { ... } +// pair starting at the first '{' in s, and a bool indicating success. +// The returned substring excludes the outer braces. +func scanBalancedBraces(s string) (string, bool) { + open := strings.Index(s, "{") + if open < 0 { + return "", false + } + depth := 0 + for i := open; i < len(s); i++ { + switch s[i] { + case '{': + depth++ + case '}': + depth-- + if depth == 0 { + return s[open+1 : i], true + } + } + } + return "", false +} diff --git a/internal/schema/pkl/project_deps_test.go b/internal/schema/pkl/project_deps_test.go new file mode 100644 index 00000000..66bc0cc8 --- /dev/null +++ b/internal/schema/pkl/project_deps_test.go @@ -0,0 +1,94 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package pkl + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParsePklProjectDeps_RemoteOnly(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "PklProject") + require.NoError(t, os.WriteFile(path, []byte(`amends "pkl:Project" + +dependencies { + ["formae"] { + uri = "package://hub.platform.engineering/plugins/pkl/schema/pkl/formae/formae@0.85.0" + } + ["aws"] { + uri = "package://hub.platform.engineering/plugins/aws/schema/pkl/aws/aws@0.1.5" + } +} +`), 0644)) + + deps, err := parsePklProjectDeps(path) + require.NoError(t, err) + assert.ElementsMatch(t, []string{ + "pkl.formae@0.85.0", + "aws.aws@0.1.5", + }, deps) +} + +func TestParsePklProjectDeps_LocalOnly(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "PklProject") + require.NoError(t, os.WriteFile(path, []byte(`amends "pkl:Project" + +dependencies { + ["aws"] = import("/home/me/.pel/formae/plugins/aws/v0.1.5/schema/pkl/PklProject") +} +`), 0644)) + + deps, err := parsePklProjectDeps(path) + require.NoError(t, err) + assert.Equal(t, []string{ + "local:aws:/home/me/.pel/formae/plugins/aws/v0.1.5/schema/pkl/PklProject", + }, deps) +} + +func TestParsePklProjectDeps_Mixed(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "PklProject") + require.NoError(t, os.WriteFile(path, []byte(`amends "pkl:Project" + +dependencies { + ["formae"] { + uri = "package://hub.platform.engineering/plugins/pkl/schema/pkl/formae/formae@0.85.0" + } + ["aws"] = import("/path/to/aws/PklProject") +} +`), 0644)) + + deps, err := parsePklProjectDeps(path) + require.NoError(t, err) + assert.ElementsMatch(t, []string{ + "pkl.formae@0.85.0", + "local:aws:/path/to/aws/PklProject", + }, deps) +} + +func TestParsePklProjectDeps_EmptyBlock(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "PklProject") + require.NoError(t, os.WriteFile(path, []byte(`amends "pkl:Project" + +dependencies { +} +`), 0644)) + + deps, err := parsePklProjectDeps(path) + require.NoError(t, err) + assert.Empty(t, deps) +} + +func TestParsePklProjectDeps_FileMissing(t *testing.T) { + _, err := parsePklProjectDeps("/no/such/file") + require.Error(t, err) +} diff --git a/internal/schema/pkl/testdata/config/test_deprecation_artifacts.pkl b/internal/schema/pkl/testdata/config/test_deprecation_artifacts.pkl new file mode 100644 index 00000000..9a13fef6 --- /dev/null +++ b/internal/schema/pkl/testdata/config/test_deprecation_artifacts.pkl @@ -0,0 +1,14 @@ +/* + * © 2025 Platform Engineering Labs Inc. + * + * SPDX-License-Identifier: FSL-1.1-ALv2 + */ + +amends "formae:/Config.pkl" + +artifacts { + url = "https://example.org/repo#stable" + username = "alice" + password = "s3cr3t" + repositories = new Listing {} +} diff --git a/internal/schema/schema.go b/internal/schema/schema.go index 41e87c3a..01ea4e76 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -38,6 +38,18 @@ type SerializeOptions struct { Colorize bool Simplified bool SchemaLocation SchemaLocation + + // LocalPluginDir is the directory to scan when SchemaLocation == SchemaLocationLocal + // and Dependencies is empty. Populated by the App from the loaded config (Config.PluginDir), + // not from an env var. Forward-compat note: PR #410 will eventually replace this single + // dir with a multi-dir list backed by discovery.SystemPluginDir + DiscoverPluginsMulti. + LocalPluginDir string + + // Dependencies, when non-empty, is a pre-resolved list of package specs (in the + // same format that PackageResolver emits — "plugin.name@version" for remote, + // "local:name:/abs/path" for local). Schema plugins MUST use these directly + // instead of doing their own discovery. + Dependencies []string } // ErrFailedToGenerateSources is returned when source code generation fails. @@ -51,7 +63,7 @@ type SchemaPlugin interface { FormaeConfig(path string) (*model.Config, error) Evaluate(path string, cmd model.Command, mode model.FormaApplyMode, props map[string]string) (*model.Forma, error) SerializeForma(resources *model.Forma, options *SerializeOptions) (string, error) - GenerateSourceCode(forma *model.Forma, targetPath string, includes []string, schemaLocation SchemaLocation) (GenerateSourcesResult, error) + GenerateSourceCode(forma *model.Forma, targetPath string, includes []string, options *SerializeOptions) (GenerateSourcesResult, error) ProjectInit(path string, include []string, schemaLocation SchemaLocation) error ProjectProperties(path string) (map[string]model.Prop, error) } diff --git a/internal/schema/schema_test.go b/internal/schema/schema_test.go index 619c1a70..3e354465 100644 --- a/internal/schema/schema_test.go +++ b/internal/schema/schema_test.go @@ -32,7 +32,7 @@ func (m *mockSchemaPlugin) Evaluate(_ string, _ model.Command, _ model.FormaAppl func (m *mockSchemaPlugin) SerializeForma(_ *model.Forma, _ *SerializeOptions) (string, error) { panic("not implemented") } -func (m *mockSchemaPlugin) GenerateSourceCode(_ *model.Forma, _ string, _ []string, _ SchemaLocation) (GenerateSourcesResult, error) { +func (m *mockSchemaPlugin) GenerateSourceCode(_ *model.Forma, _ string, _ []string, _ *SerializeOptions) (GenerateSourcesResult, error) { panic("not implemented") } func (m *mockSchemaPlugin) ProjectInit(_ string, _ []string, _ SchemaLocation) error { diff --git a/internal/schema/yaml/yaml.go b/internal/schema/yaml/yaml.go index ea9ca0fd..facab2b0 100644 --- a/internal/schema/yaml/yaml.go +++ b/internal/schema/yaml/yaml.go @@ -140,7 +140,7 @@ func highlight(code []byte) ([]byte, error) { return buf.Bytes(), nil } -func (j YAML) GenerateSourceCode(forma *model.Forma, targetPath string, includes []string, schemaLocation schema.SchemaLocation) (schema.GenerateSourcesResult, error) { +func (j YAML) GenerateSourceCode(forma *model.Forma, targetPath string, includes []string, options *schema.SerializeOptions) (schema.GenerateSourcesResult, error) { return schema.GenerateSourcesResult{}, errors.ErrUnsupported } diff --git a/pkg/api/model/errors.go b/pkg/api/model/errors.go index 996dd518..cef842e2 100644 --- a/pkg/api/model/errors.go +++ b/pkg/api/model/errors.go @@ -28,6 +28,11 @@ const ( StackDeletedDuringApply APIError = "StackDeletedDuringApply" ReconcilePolicyRequired APIError = "ReconcilePolicyRequired" NonPortableResources APIError = "NonPortableResources" + PluginNotFound APIError = "PluginNotFound" + PluginVersionNotFound APIError = "PluginVersionNotFound" + PluginDependencyConflict APIError = "PluginDependencyConflict" + PluginRepositoryUnreachable APIError = "PluginRepositoryUnreachable" + PluginSignatureInvalid APIError = "PluginSignatureInvalid" ) type ErrorResponse[T any] struct { @@ -192,3 +197,19 @@ type TargetHasResourcesError struct { func (e TargetHasResourcesError) Error() string { return fmt.Sprintf("target %s cannot be deleted: has %d deployed resources", e.TargetLabel, e.ResourceCount) } + +type PluginNotFoundError struct { + Name string `json:"Name"` +} + +func (e PluginNotFoundError) Error() string { + return fmt.Sprintf("plugin %q not found", e.Name) +} + +type PluginDependencyConflictError struct { + Message string `json:"Message"` +} + +func (e PluginDependencyConflictError) Error() string { + return fmt.Sprintf("plugin dependency conflict: %s", e.Message) +} diff --git a/pkg/api/model/go.mod b/pkg/api/model/go.mod index 50de96cc..2d5104ae 100644 --- a/pkg/api/model/go.mod +++ b/pkg/api/model/go.mod @@ -2,13 +2,19 @@ module github.com/platform-engineering-labs/formae/pkg/api/model go 1.26 -require github.com/platform-engineering-labs/formae/pkg/model v0.0.0-20260120024139-dfbbaa3804a2 +require ( + github.com/platform-engineering-labs/formae/pkg/model v0.0.0-20260120024139-dfbbaa3804a2 + github.com/stretchr/testify v1.11.1 +) require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/theory/jsonpath v0.10.2 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) replace github.com/platform-engineering-labs/formae/pkg/model => ../../model diff --git a/pkg/api/model/go.sum b/pkg/api/model/go.sum index 31f0b67e..3c89bb56 100644 --- a/pkg/api/model/go.sum +++ b/pkg/api/model/go.sum @@ -1,7 +1,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/theory/jsonpath v0.10.2 h1:i8GeMxnD6ftNWeSeaGb/Eb8XghGjsas1eDizaQNupuE= @@ -13,5 +19,8 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/api/model/plugin_test.go b/pkg/api/model/plugin_test.go new file mode 100644 index 00000000..19780d7f --- /dev/null +++ b/pkg/api/model/plugin_test.go @@ -0,0 +1,62 @@ +//go:build unit + +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +package model + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPluginJSONRoundTrip(t *testing.T) { + p := Plugin{ + Name: "aws", + Type: "resource", + Namespace: "AWS", + Category: "cloud", + Summary: "AWS plugin", + InstalledVersion: "1.4.2", + AvailableVersions: []string{"1.4.2", "1.3.1"}, + ManagedBy: "standard", + } + data, err := json.Marshal(p) + require.NoError(t, err) + var back Plugin + require.NoError(t, json.Unmarshal(data, &back)) + assert.Equal(t, p, back) +} + +func TestInstallPluginsRequestJSONRoundTrip(t *testing.T) { + req := InstallPluginsRequest{ + Packages: []PackageRef{ + {Name: "aws", Version: "1.4.2"}, + {Name: "azure"}, + }, + } + data, err := json.Marshal(req) + require.NoError(t, err) + var back InstallPluginsRequest + require.NoError(t, json.Unmarshal(data, &back)) + assert.Equal(t, req, back) +} + +func TestInstallPluginsResponseJSONRoundTrip(t *testing.T) { + resp := InstallPluginsResponse{ + Operations: []PluginOperation{ + {Name: "aws", Version: "1.4.2", Action: "install"}, + }, + RequiresRestart: true, + Warnings: []string{"Restart the agent"}, + } + data, err := json.Marshal(resp) + require.NoError(t, err) + var back InstallPluginsResponse + require.NoError(t, json.Unmarshal(data, &back)) + assert.Equal(t, resp, back) +} diff --git a/pkg/api/model/types.go b/pkg/api/model/types.go index 78fb485a..bc67dbb7 100644 --- a/pkg/api/model/types.go +++ b/pkg/api/model/types.go @@ -195,3 +195,86 @@ type ForceCheckTTLResponse struct { ExpiredStacks []string `json:"expired_stacks"` CommandIDs []string `json:"command_ids,omitempty"` } + +// Plugin describes a single plugin, used by the list and info endpoints. +type Plugin struct { + Name string `json:"name"` + Kind string `json:"kind,omitempty"` + Type string `json:"type"` + Namespace string `json:"namespace,omitempty"` + Category string `json:"category,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` + Publisher string `json:"publisher,omitempty"` + License string `json:"license,omitempty"` + InstalledVersion string `json:"installedVersion,omitempty"` + AvailableVersions []string `json:"availableVersions,omitempty"` + // LocalPath is the absolute path on the agent's filesystem to the + // plugin's PklProject file (containing the plugin's PKL schema). + // Populated by the discovery scan when the plugin is installed + // locally; empty when no on-disk install is found. Used by the CLI's + // --schema-location local flow to import schemas via PklProject.deps + // rather than fetching from the hub. Same-box only — the path is + // only meaningful when the CLI shares a filesystem with the agent. + LocalPath string `json:"localPath,omitempty"` + + Channel string `json:"channel,omitempty"` + Frozen bool `json:"frozen,omitempty"` + ManagedBy string `json:"managedBy,omitempty"` + LoadStatus string `json:"loadStatus,omitempty"` + Metadata map[string]map[string]string `json:"metadata,omitempty"` +} + +// PluginOperation describes a single operation performed on a plugin. +type PluginOperation struct { + Name string `json:"name"` + Type string `json:"type,omitempty"` + Version string `json:"version,omitempty"` + Action string `json:"action"` // "install" | "remove" | "noop" +} + +// PackageRef identifies a plugin package, optionally at a specific version. +type PackageRef struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` +} + +type ListPluginsResponse struct { + Plugins []Plugin `json:"plugins"` +} + +type GetPluginResponse struct { + Plugin Plugin `json:"plugin"` +} + +type InstallPluginsRequest struct { + Packages []PackageRef `json:"packages"` + Channel string `json:"channel,omitempty"` +} + +type InstallPluginsResponse struct { + Operations []PluginOperation `json:"operations"` + RequiresRestart bool `json:"requiresRestart"` + Warnings []string `json:"warnings,omitempty"` +} + +type UninstallPluginsRequest struct { + Packages []PackageRef `json:"packages"` +} + +type UninstallPluginsResponse struct { + Operations []PluginOperation `json:"operations"` + RequiresRestart bool `json:"requiresRestart"` + Warnings []string `json:"warnings,omitempty"` +} + +type UpgradePluginsRequest struct { + Packages []PackageRef `json:"packages,omitempty"` + Channel string `json:"channel,omitempty"` +} + +type UpgradePluginsResponse struct { + Operations []PluginOperation `json:"operations"` + RequiresRestart bool `json:"requiresRestart"` + Warnings []string `json:"warnings,omitempty"` +} diff --git a/pkg/model/config.go b/pkg/model/config.go index caf18728..2525382f 100644 --- a/pkg/model/config.go +++ b/pkg/model/config.go @@ -216,10 +216,27 @@ type APIConfig struct { Port int } +// RepositoryType discriminates orbital repositories by purpose. +type RepositoryType string + +const ( + RepositoryTypeBinary RepositoryType = "binary" + RepositoryTypeFormaePlugin RepositoryType = "formae-plugin" +) + +// Repository points to a single orbital repository. +type Repository struct { + URI url.URL + Type RepositoryType +} + type ArtifactConfig struct { URL url.URL Username string Password string + // Repositories is the canonical list. When empty and URL is set, + // the translation layer populates it with a single binary entry. + Repositories []Repository } type CliConfig struct { diff --git a/pkg/plugin/discovery/migrate.go b/pkg/plugin/discovery/migrate.go index 9cc2e0ac..b1f1bd38 100644 --- a/pkg/plugin/discovery/migrate.go +++ b/pkg/plugin/discovery/migrate.go @@ -15,7 +15,16 @@ const migratedMarker = ".migrated-v1" // CleanStaleDevPlugins removes plugins from devDir that also exist in systemDir. // This is a one-time migration: once run, a marker file is written to devDir to // prevent re-running on subsequent startups. +// +// No-op when devDir and systemDir resolve to the same path: every "stale dev" +// match would be deleting the only copy. This happens in test setups that +// deliberately point cfg.PluginDir at the system install directory (e.g. +// orbital-installed plugins surfaced through the same path as a "dev" view). func CleanStaleDevPlugins(systemDir, devDir string) error { + if same, err := samePath(systemDir, devDir); err == nil && same { + return nil + } + markerPath := filepath.Join(devDir, migratedMarker) if _, err := os.Stat(markerPath); err == nil { return nil // already migrated @@ -59,3 +68,18 @@ func CleanStaleDevPlugins(systemDir, devDir string) error { return os.WriteFile(markerPath, []byte("done"), 0644) } + +// samePath reports whether a and b refer to the same directory on disk. +// Tries os.SameFile (handles symlinks, relative-vs-absolute, ./ prefixes, +// etc.) and falls back to string comparison if either path can't be stat'd. +func samePath(a, b string) (bool, error) { + aInfo, errA := os.Stat(a) + bInfo, errB := os.Stat(b) + if errA == nil && errB == nil { + return os.SameFile(aInfo, bInfo), nil + } + if errA != nil { + return false, errA + } + return false, errB +} diff --git a/pkg/plugin/manifest.go b/pkg/plugin/manifest.go index e0dd127e..6818f3c7 100644 --- a/pkg/plugin/manifest.go +++ b/pkg/plugin/manifest.go @@ -37,6 +37,12 @@ type Manifest struct { // MinFormaeVersion is the minimum formae version this plugin supports // Used for compatibility checking and matrix testing MinFormaeVersion string `json:"minFormaeVersion"` + + // Summary is a short one-liner description for CLI listings (optional) + Summary string `json:"summary,omitempty"` + + // Category is a UI filter tag, e.g. "cloud", "auth", "config" (optional) + Category string `json:"category,omitempty"` } // IsAuthPlugin returns true if this manifest describes an auth plugin. diff --git a/pkg/plugin/manifest_test.go b/pkg/plugin/manifest_test.go index 15372edf..b275247c 100644 --- a/pkg/plugin/manifest_test.go +++ b/pkg/plugin/manifest_test.go @@ -64,6 +64,47 @@ output { renderer = new JsonRenderer {} } assert.Equal(t, "DirTest", manifest.Namespace) } +func TestReadManifest_ParsesSummaryAndCategory(t *testing.T) { + tempDir := t.TempDir() + manifestContent := ` +name = "summary-test" +version = "1.0.0" +namespace = "Test" +license = "MIT" +minFormaeVersion = "0.85.0" +summary = "A short one-liner for testing" +category = "cloud" +output { renderer = new JsonRenderer {} } +` + err := os.WriteFile(filepath.Join(tempDir, "formae-plugin.pkl"), []byte(manifestContent), 0644) + require.NoError(t, err) + + manifest, err := ReadManifestFromDir(tempDir) + require.NoError(t, err) + assert.Equal(t, "A short one-liner for testing", manifest.Summary) + assert.Equal(t, "cloud", manifest.Category) +} + +func TestReadManifest_SummaryAndCategoryOptional(t *testing.T) { + tempDir := t.TempDir() + manifestContent := ` +name = "no-extras" +version = "1.0.0" +namespace = "Test" +license = "MIT" +minFormaeVersion = "0.85.0" +output { renderer = new JsonRenderer {} } +` + err := os.WriteFile(filepath.Join(tempDir, "formae-plugin.pkl"), []byte(manifestContent), 0644) + require.NoError(t, err) + + manifest, err := ReadManifestFromDir(tempDir) + require.NoError(t, err) + assert.Equal(t, "", manifest.Summary) + assert.Equal(t, "", manifest.Category) + assert.NoError(t, manifest.Validate()) +} + func TestManifest_Validate_RequiresAllFields(t *testing.T) { tests := []struct { name string @@ -96,10 +137,15 @@ func TestManifest_Validate_RequiresAllFields(t *testing.T) { expectError: "minFormaeVersion is required", }, { - name: "valid manifest", + name: "valid manifest without summary or category", manifest: Manifest{Name: "test", Version: "1.0.0", Namespace: "Test", License: "MIT", MinFormaeVersion: "0.80.0"}, expectError: "", }, + { + name: "valid manifest with summary and category", + manifest: Manifest{Name: "test", Version: "1.0.0", Namespace: "Test", License: "MIT", MinFormaeVersion: "0.80.0", Summary: "A plugin", Category: "cloud"}, + expectError: "", + }, } for _, tt := range tests { diff --git a/tests/blackbox/testmain_test.go b/tests/blackbox/testmain_test.go index 17ee2228..b5e6cb4b 100644 --- a/tests/blackbox/testmain_test.go +++ b/tests/blackbox/testmain_test.go @@ -18,13 +18,25 @@ func TestMain(m *testing.M) { // Find project root (walk up to go.mod) projectRoot := findProjectRoot() - // Build formae binary once for all blackbox tests + // Build formae binary once for all blackbox tests. The binary lives at + // /bin/formae so orbital's tree-root derivation + // (filepath.Dir(filepath.Dir(binPath))) resolves to , where we + // then create an empty .ops/ directory. That satisfies orbital's + // Ready() check (it only tests for the directory's existence) without + // scaffolding a real signed orbital tree, so the agent's plugin manager + // initializes cleanly at startup. Production builds installed via + // setup.sh have a fully populated orbital tree at /opt/pel and follow + // the same path naturally. tmpDir, err := os.MkdirTemp("", "formae-blackbox-*") if err != nil { log.Fatalf("failed to create temp dir: %v", err) } - formaeBinary = filepath.Join(tmpDir, "formae") + formaeBinary = filepath.Join(tmpDir, "bin", "formae") + if err := os.MkdirAll(filepath.Dir(formaeBinary), 0755); err != nil { + os.RemoveAll(tmpDir) + log.Fatalf("Failed to create binary dir: %v", err) + } cmd := exec.Command("go", "build", "-o", formaeBinary, "./cmd/formae") cmd.Dir = projectRoot cmd.Stdout = os.Stderr @@ -33,6 +45,10 @@ func TestMain(m *testing.M) { os.RemoveAll(tmpDir) log.Fatalf("Failed to build formae binary: %v", err) } + if err := os.MkdirAll(filepath.Join(tmpDir, ".ops"), 0755); err != nil { + os.RemoveAll(tmpDir) + log.Fatalf("Failed to create orbital tree marker: %v", err) + } code := m.Run() os.RemoveAll(tmpDir) diff --git a/tests/e2e/go/agent.go b/tests/e2e/go/agent.go index c4722005..3bc19a9c 100644 --- a/tests/e2e/go/agent.go +++ b/tests/e2e/go/agent.go @@ -146,12 +146,21 @@ func StartAgent(t *testing.T, binaryPath string, opts ...AgentOption) *Agent { }`, options.authUsername, options.authPassword) } + // Intentionally no pluginDir override. cfg.PluginDir defaults to + // ~/.pel/formae/plugins (empty in CI) and the multi-source plugin + // discovery added in the discovery refactor finds orbital-installed + // plugins via SystemPluginDir(binPath) without help. The CLI's + // extract / project init paths now query the agent for installed + // plugin versions instead of scanning local dirs, so a single + // pluginDir on the CLI box no longer matters. + pluginDirBlock := "" + configContent := fmt.Sprintf(`/* * Auto-generated e2e test configuration */ amends "formae:/Config.pkl" -%s +%s%s agent { server { port = %d @@ -182,7 +191,7 @@ cli { } disableUsageReporting = true%s } -`, options.pklImports, port, dbPath, discoveryEnabled, options.discoveryInterval, resourceTypesBlock, logPath, agentAuthBlock, options.resourcePluginsBlock, port, cliAuthBlock) +`, options.pklImports, pluginDirBlock, port, dbPath, discoveryEnabled, options.discoveryInterval, resourceTypesBlock, logPath, agentAuthBlock, options.resourcePluginsBlock, port, cliAuthBlock) if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { t.Fatalf("failed to write agent config: %v", err) diff --git a/tests/e2e/go/eval_test.go b/tests/e2e/go/eval_test.go new file mode 100644 index 00000000..995f4cff --- /dev/null +++ b/tests/e2e/go/eval_test.go @@ -0,0 +1,135 @@ +// © 2025 Platform Engineering Labs Inc. +// +// SPDX-License-Identifier: FSL-1.1-ALv2 + +//go:build e2e + +package e2e_test + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// TestEvalDefault verifies that `formae eval` against a forma file using a +// resource plugin produces valid JSON output. Locks down the post-fix +// behavior where eval queries the agent for installed plugin versions and +// resolves remote PKL packages from the hub. +// +// Regression coverage for the bug fixed in 79063e57: pre-fix, eval scanned +// the CLI's local plugin dir which could differ from the agent's path, +// producing malformed dep strings (`aws.aws@`, no version). +func TestEvalDefault(t *testing.T) { + bin := FormaeBinary(t) + agent := StartAgent(t, bin) + cli := NewFormaeCLI(bin, agent.ConfigPath(), agent.Port()) + + fixture := filepath.Join(fixturesDir(t), "reconcile_apply_aws.pkl") + stdout := cli.Eval(t, fixture) + if len(stdout) == 0 { + t.Fatal("formae eval returned empty output") + } + + var parsed map[string]any + if err := json.Unmarshal(stdout, &parsed); err != nil { + t.Fatalf("eval output is not valid JSON: %v\noutput: %s", err, string(stdout)) + } + + if _, ok := parsed["Stacks"]; !ok { + t.Errorf("eval output missing Stacks field; got keys: %v", keysOf(parsed)) + } + if _, ok := parsed["Resources"]; !ok { + t.Errorf("eval output missing Resources field; got keys: %v", keysOf(parsed)) + } +} + +// TestEvalSchemaLocationLocal verifies that `formae eval --schema-location +// local` accepts the flag, resolves plugin schemas from the agent's local +// filesystem paths, and produces equivalent output to the default (remote) +// path. Same-box only — the flag is intended for the developer workflow +// where CLI and agent share a filesystem. +// +// Red until the --schema-location flag is wired into eval and the agent's +// plugin-info response is extended with localPath. +func TestEvalSchemaLocationLocal(t *testing.T) { + bin := FormaeBinary(t) + agent := StartAgent(t, bin) + cli := NewFormaeCLI(bin, agent.ConfigPath(), agent.Port()) + + fixture := filepath.Join(fixturesDir(t), "reconcile_apply_aws.pkl") + stdout := cli.Eval(t, fixture, "--schema-location", "local") + if len(stdout) == 0 { + t.Fatal("formae eval --schema-location local returned empty output") + } + + var parsed map[string]any + if err := json.Unmarshal(stdout, &parsed); err != nil { + t.Fatalf("eval output is not valid JSON: %v\noutput: %s", err, string(stdout)) + } + + if _, ok := parsed["Stacks"]; !ok { + t.Errorf("eval output missing Stacks field; got keys: %v", keysOf(parsed)) + } +} + +// TestExtractSchemaLocationLocal verifies that `formae extract --schema-location +// local` writes a PklProject whose dependencies are local file imports +// (`import("...")`) rather than remote hub URIs (`uri = "package://..."`). +// Same-box only. +// +// Red until the --schema-location flag is wired into extract. +func TestExtractSchemaLocationLocal(t *testing.T) { + bin := FormaeBinary(t) + agent := StartAgent(t, bin) + cli := NewFormaeCLI(bin, agent.ConfigPath(), agent.Port()) + + fixture := filepath.Join(fixturesDir(t), "extract_schema_local_aws.pkl") + commandTimeout := 2 * time.Minute + + cmdID := cli.Apply(t, "reconcile", fixture) + result := cli.WaitForCommand(t, cmdID, commandTimeout) + RequireCommandSuccess(t, result) + + t.Cleanup(func() { + destroyID := cli.Destroy(t, fixture) + destroyResult := cli.WaitForCommand(t, destroyID, commandTimeout) + if destroyResult.State != "Success" && destroyResult.State != "NoOp" { + t.Logf("cleanup destroy ended in state %s", destroyResult.State) + } + }) + + extractDir := t.TempDir() + extractedPath := filepath.Join(extractDir, "extracted.pkl") + cli.ExtractToFile(t, "stack:e2e-extract-schema-local-aws", extractedPath, "--schema-location", "local") + + pklProjectPath := filepath.Join(extractDir, "PklProject") + data, err := os.ReadFile(pklProjectPath) + if err != nil { + t.Fatalf("PklProject not found at %s: %v", pklProjectPath, err) + } + content := string(data) + + // aws should resolve via a local import(); formae core stays remote + // (the agent does not surface its own PKL schema as a local path, + // and dev workflows typically test against a published formae). + if !strings.Contains(content, `["aws"] = import(`) { + t.Errorf("PklProject does not contain a local import() for aws under --schema-location local.\nContent:\n%s", content) + } + if strings.Contains(content, `["aws"] {`) && strings.Contains(content, "package://hub.platform.engineering/plugins/aws") { + t.Errorf("PklProject still resolves aws via remote package:// URI under --schema-location local.\nContent:\n%s", content) + } +} + +// keysOf returns the keys of a map, used for clearer error messages when an +// expected field is missing. +func keysOf(m map[string]any) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return keys +} diff --git a/tests/e2e/go/fixtures/extract_schema_local_aws.pkl b/tests/e2e/go/fixtures/extract_schema_local_aws.pkl new file mode 100644 index 00000000..94dabea4 --- /dev/null +++ b/tests/e2e/go/fixtures/extract_schema_local_aws.pkl @@ -0,0 +1,48 @@ +/* + * © 2025 Platform Engineering Labs Inc. + * + * SPDX-License-Identifier: FSL-1.1-ALv2 + */ + +amends "@formae/forma.pkl" +import "@formae/formae.pkl" + +import "@aws/aws.pkl" + +import "@aws/iam/role.pkl" + +local parentRole = new role.Role { + label = "e2e-extract-schema-local-role" + roleName = "formae-e2e-extract-schema-local-role" + description = "e2e extract --schema-location local test role" + assumeRolePolicyDocument = new Dynamic { + ["Version"] = "2012-10-17" + ["Statement"] = new Listing { + new Dynamic { + ["Effect"] = "Allow" + ["Principal"] = new Dynamic { + ["Service"] = "lambda.amazonaws.com" + } + ["Action"] = "sts:AssumeRole" + } + } + } +} + +local stackName = "e2e-extract-schema-local-aws" + +forma { + new formae.Stack { + label = stackName + description = "E2E extract --schema-location local test fixture" + } + + new formae.Target { + label = "e2e-aws-target" + config = new aws.Config { + region = "us-west-2" + } + } + + parentRole +} diff --git a/tests/e2e/go/formae.go b/tests/e2e/go/formae.go index 5bfe7cd3..132850c6 100644 --- a/tests/e2e/go/formae.go +++ b/tests/e2e/go/formae.go @@ -11,6 +11,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" "os/exec" "testing" "time" @@ -270,8 +271,9 @@ func (f *FormaeCLI) Inventory(t *testing.T, args ...string) []Resource { // ExtractToFile runs `formae extract` with the given query and writes the // resulting PKL to targetPath. The --yes flag is set to overwrite without -// prompting. -func (f *FormaeCLI) ExtractToFile(t *testing.T, query string, targetPath string) { +// prompting. Additional flags can be passed via extraArgs (e.g. +// "--schema-location", "local"). +func (f *FormaeCLI) ExtractToFile(t *testing.T, query string, targetPath string, extraArgs ...string) { t.Helper() args := []string{ @@ -279,12 +281,39 @@ func (f *FormaeCLI) ExtractToFile(t *testing.T, query string, targetPath string) "--config", f.configPath, "--query", query, "--yes", - targetPath, } + args = append(args, extraArgs...) + args = append(args, targetPath) f.run(t, args...) } +// Eval runs `formae eval` against the given forma fixture and returns stdout +// as bytes. The machine consumer + json output schema are set so the result +// is parseable. +// +// The fixture path is placed immediately after the subcommand because +// IsDynamicCommand (cmd.go) iterates os.Args looking for the first argument +// containing a supported file extension and treats it as the forma file. +// If --config (which points at formae.conf.pkl) appeared first, it would be +// mis-detected as the forma file and the PKL evaluator would refuse to +// load formae:/Config.pkl. Apply's helper uses the same ordering for the +// same reason. +func (f *FormaeCLI) Eval(t *testing.T, fixturePath string, extraArgs ...string) []byte { + t.Helper() + + args := []string{ + "eval", + fixturePath, + "--config", f.configPath, + "--output-consumer", "machine", + "--output-schema", "json", + } + args = append(args, extraArgs...) + + return f.run(t, args...) +} + // StatusCommand queries the status of a specific command by ID. func (f *FormaeCLI) StatusCommand(t *testing.T, commandID string) CommandResult { t.Helper() @@ -481,7 +510,11 @@ func (f *FormaeCLI) DestroyExpectError(t *testing.T, fixturePath string, extraAr } // ProjectInit runs `formae project init` to generate a new project in the -// given directory. This is a CLI-only command that does not require an agent. +// given directory. The CLI consults the agent for installed plugin versions +// (used to pin remote schema URIs in the generated PklProject), so an agent +// must be running and the FormaeCLI must have been constructed with its +// config path. --plugin-dir is only needed when an include uses the @local +// suffix. func (f *FormaeCLI) ProjectInit(t *testing.T, dir string, includes ...string) { t.Helper() @@ -490,12 +523,16 @@ func (f *FormaeCLI) ProjectInit(t *testing.T, dir string, includes ...string) { "--schema", "pkl", "--yes", } + if f.configPath != "" { + args = append(args, "--config", f.configPath) + } + if pluginDir := os.Getenv("FORMAE_PLUGIN_DIR"); pluginDir != "" { + args = append(args, "--plugin-dir", pluginDir) + } for _, inc := range includes { args = append(args, "--include", inc) } - // ProjectInit does not need --config (no agent), so use exec.Command - // directly instead of f.run() which always adds --config. cmd := exec.Command(f.binaryPath, args...) var stdout, stderr bytes.Buffer cmd.Stdout = &stdout diff --git a/tests/e2e/go/project_init_test.go b/tests/e2e/go/project_init_test.go index 4dbd1c69..0a0e0e2c 100644 --- a/tests/e2e/go/project_init_test.go +++ b/tests/e2e/go/project_init_test.go @@ -16,8 +16,12 @@ import ( func TestProjectInit(t *testing.T) { bin := FormaeBinary(t) - // No agent needed — this is a CLI-only test. - cli := NewFormaeCLI(bin, "", 0) + // project init resolves non-@local plugin versions via the agent's + // installed-plugins endpoint after the multi-source plugin-discovery + // refactor — orbital-installed plugins live with the agent, not on + // the CLI box. Start an agent so the version lookup succeeds. + agent := StartAgent(t, bin) + cli := NewFormaeCLI(bin, agent.ConfigPath(), agent.Port()) // Step 1: Create a temp directory for the new project. dir := t.TempDir() diff --git a/tests/e2e/go/setup_pkl.sh b/tests/e2e/go/setup_pkl.sh index e242798e..fdab4fee 100755 --- a/tests/e2e/go/setup_pkl.sh +++ b/tests/e2e/go/setup_pkl.sh @@ -14,7 +14,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)" FIXTURES_DIR="$SCRIPT_DIR/fixtures" -PLUGINS_DIR="$HOME/.pel/formae/plugins" +PLUGINS_DIR="${FORMAE_PLUGIN_DIR:-$HOME/.pel/formae/plugins}" PKLPROJECT_PATH="$FIXTURES_DIR/PklProject" # Ensure version.semver exists (needed by formae PklProject). @@ -51,11 +51,11 @@ hub_uri() { if [[ -z "$plugin_dir" ]] || [[ ! -f "$plugin_dir/schema/pkl/PklProject" ]]; then if [[ "$required" == "true" ]]; then - echo "ERROR: $alias plugin not found at $PLUGINS_DIR/$ns_dir/" - echo "Run 'make install-external-plugins' first." + echo "ERROR: $alias plugin not found at $PLUGINS_DIR/$ns_dir/" >&2 + echo "Install plugins via 'formae plugin install $alias' against an agent whose orbital tree shares this directory, or set FORMAE_PLUGIN_DIR to a tree that has it." >&2 exit 1 else - echo "WARN: $alias plugin not found at $PLUGINS_DIR/$ns_dir/ — related E2E tests will be skipped" >&2 + echo "WARN: $alias plugin not found at $PLUGINS_DIR/$ns_dir/ — fixtures importing this plugin will fail to evaluate" >&2 return fi fi @@ -67,25 +67,29 @@ hub_uri() { echo " [\"$alias\"] { uri = \"$base_uri@$version\" }" } -# Resolve plugin schemas from the hub. AWS and Azure are required; -# compose and grafana are optional (needed for target resolvable tests). -AWS_DEP=$(hub_uri "aws" "aws" true) -AZURE_DEP=$(hub_uri "azure" "azure" true) -COMPOSE_DEP=$(hub_uri "docker" "compose" false) +# Resolve plugin schemas from the hub. All plugins are optional at this +# level — the e2e workflow installs only the plugins each test needs (matrix +# .plugins), so a single setup_pkl.sh run is shared across tests with +# different plugin sets. Fixtures that import a plugin not installed will +# fail to evaluate with a clear PKL error, which is the right failure mode. +AWS_DEP=$(hub_uri "aws" "aws" false) +AZURE_DEP=$(hub_uri "azure" "azure" false) +COMPOSE_DEP=$(hub_uri "compose" "compose" false) GRAFANA_DEP=$(hub_uri "grafana" "grafana" false) # Generate PklProject. Both formae core and plugins are pinned via hub URIs # — matches a real user's setup, and avoids the PKL type-identity split that # would happen if the fixture's formae and a plugin's formae were declared -# at different URIs. +# at different URIs. Each plugin dep is included only if its hub_uri call +# returned non-empty (i.e. the plugin is installed in $PLUGINS_DIR). cat > "$PKLPROJECT_PATH" << EOF amends "pkl:Project" dependencies { ["formae"] { uri = "$FORMAE_URI" } -$AWS_DEP -$AZURE_DEP -${COMPOSE_DEP:+$COMPOSE_DEP +${AWS_DEP:+$AWS_DEP +}${AZURE_DEP:+$AZURE_DEP +}${COMPOSE_DEP:+$COMPOSE_DEP }${GRAFANA_DEP:+$GRAFANA_DEP }} EOF