test-validate-metadata.yaml

name: Test Validate Metadata
on:
  workflow_dispatch:
  push:
    paths:
      - validate-metadata/**

env:
  TEST_DIR: ${{ github.workspace }}/validate-metadata/test
  IMAGE_REPOSITORY_PREFIX: ghcr.io/test-org/test-repo
  TARGET_BACKSTAGE_VERSION: 1.42.5

jobs:
  test-validation-pass:
    name: Test Validation (Should Pass)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/pass
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation passed
        env:
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;

          assert.equal(VALIDATION_PASSED, 'true');
          assert.equal(VALIDATION_ERROR_COUNT, '0');
          assert.equal(VALIDATION_ERRORS, '[]');

  test-validation-fail:
    name: Test Validation (Should Fail)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        continue-on-error: true
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/fail
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation failed with expected errors
        env:
          STEP_OUTCOME: ${{ steps.validate.outcome }}
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { STEP_OUTCOME, VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;
          const errors = JSON.parse(VALIDATION_ERRORS);

          function assertError(field, expectedError) {
            const error = errors.find(e => e.field === field);
            assert.ok(error, `Missing ${field} error`);
            assert.deepEqual(error, expectedError);
          }

          assert.equal(STEP_OUTCOME, 'failure');
          assert.equal(VALIDATION_PASSED, 'false');
          assert.equal(VALIDATION_ERROR_COUNT, '4');

          assertError('version', {
            kind: 'mismatch',
            file: 'test-fail.yaml',
            field: 'version',
            expected: '1.0.0',
            actual: '9.9.9',
            message: 'Version mismatch: expected "1.0.0" but got "9.9.9"'
          });

          assertError('dynamicArtifact.tag', {
            kind: 'mismatch',
            file: 'test-fail.yaml',
            field: 'dynamicArtifact.tag',
            expected: 'bs_1.42.5__1.0.0',
            actual: 'wrong_tag_1.0.0',
            message: 'OCI tag mismatch: expected "bs_1.42.5__1.0.0" but got "wrong_tag_1.0.0"'
          });

          assertError('dynamicArtifact.reference', {
            kind: 'mismatch',
            file: 'test-fail.yaml',
            field: 'dynamicArtifact.reference',
            expected: 'oci://ghcr.io/test-org/test-repo/test-org-plugin-test',
            actual: 'oci://ghcr.io/wrong-org/wrong-repo/wrong-image',
            message: 'OCI reference mismatch: expected "oci://ghcr.io/test-org/test-repo/test-org-plugin-test" but got "oci://ghcr.io/wrong-org/wrong-repo/wrong-image"'
          });

          assertError('backstage.supportedVersions', {
            kind: 'mismatch',
            file: 'test-fail.yaml',
            field: 'backstage.supportedVersions',
            expected: '1.42.x (from dist-dynamic/package.json: 1.42.5)',
            actual: '1.99.0',
            message: 'Backstage supportedVersions mismatch: expected "1.42.x" but got "1.99.x"'
          });

  test-validation-invalid-yaml:
    name: Test Validation (Invalid YAML)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        continue-on-error: true
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/invalid-yaml
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation failed with parse error
        env:
          STEP_OUTCOME: ${{ steps.validate.outcome }}
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { STEP_OUTCOME, VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;
          const [error] = JSON.parse(VALIDATION_ERRORS);

          assert.equal(STEP_OUTCOME, 'failure');
          assert.equal(VALIDATION_PASSED, 'false');
          assert.equal(VALIDATION_ERROR_COUNT, '1');

          assert.deepEqual(error, {
            kind: 'parse-error',
            file: 'test-invalid-yaml.yaml',
            message: 'Failed to parse YAML'
          });

  test-validation-missing-spec:
    name: Test Validation (Missing Spec)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        continue-on-error: true
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/missing-spec
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation failed with missing spec error
        env:
          STEP_OUTCOME: ${{ steps.validate.outcome }}
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { STEP_OUTCOME, VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;
          const [error] = JSON.parse(VALIDATION_ERRORS);

          assert.equal(STEP_OUTCOME, 'failure');
          assert.equal(VALIDATION_PASSED, 'false');
          assert.equal(VALIDATION_ERROR_COUNT, '1');

          assert.deepEqual(error, {
            kind: 'missing-field',
            file: 'test-missing-spec.yaml',
            field: 'spec',
            message: 'Missing required field: spec'
          });

  test-validation-unknown-package:
    name: Test Validation (Unknown Package)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        continue-on-error: true
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/unknown-package
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation failed with unknown package error
        env:
          STEP_OUTCOME: ${{ steps.validate.outcome }}
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { STEP_OUTCOME, VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;
          const [error] = JSON.parse(VALIDATION_ERRORS);

          assert.equal(STEP_OUTCOME, 'failure');
          assert.equal(VALIDATION_PASSED, 'false');
          assert.equal(VALIDATION_ERROR_COUNT, '1');

          assert.deepEqual(error, {
            kind: 'unknown-package',
            file: 'test-unknown-package.yaml',
            packageName: '@unknown-org/plugin-that-does-not-exist',
            message: 'Package "@unknown-org/plugin-that-does-not-exist" not found in plugins-list.yaml'
          });

  test-validation-missing-packagename:
    name: Test Validation (Missing Package Name)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        continue-on-error: true
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/missing-packagename
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation failed with missing packageName error
        env:
          STEP_OUTCOME: ${{ steps.validate.outcome }}
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { STEP_OUTCOME, VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;
          const [error] = JSON.parse(VALIDATION_ERRORS);

          assert.equal(STEP_OUTCOME, 'failure');
          assert.equal(VALIDATION_PASSED, 'false');
          assert.equal(VALIDATION_ERROR_COUNT, '1');

          assert.deepEqual(error, {
            kind: 'missing-field',
            file: 'test-missing-packagename.yaml',
            field: 'packageName',
            message: 'Missing required field: packageName'
          });

  test-validation-missing-files:
    name: Test Validation (Missing Required Files)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        continue-on-error: true
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/missing-files
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation failed with missing files errors
        env:
          STEP_OUTCOME: ${{ steps.validate.outcome }}
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { STEP_OUTCOME, VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS, TEST_DIR } = process.env;
          const [metadataDirError, pluginsListError] = JSON.parse(VALIDATION_ERRORS);

          assert.equal(STEP_OUTCOME, 'failure');
          assert.equal(VALIDATION_PASSED, 'false');
          assert.equal(VALIDATION_ERROR_COUNT, '2');

          assert.deepEqual(metadataDirError, {
            kind: 'missing-file',
            file: 'metadata',
            message: `Metadata directory not found at ${TEST_DIR}/cases/missing-files/metadata`
          });

          assert.deepEqual(pluginsListError, {
            kind: 'missing-file',
            file: 'plugins-list.yaml',
            message: `plugins-list.yaml not found at ${TEST_DIR}/cases/missing-files/plugins-list.yaml`
          });

  test-validation-invalid-plugins-list:
    name: Test Validation (Invalid plugins-list.yaml)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata
        id: validate
        continue-on-error: true
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/invalid-plugins-list
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}

      - name: Verify validation failed with invalid plugins-list error
        env:
          STEP_OUTCOME: ${{ steps.validate.outcome }}
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { STEP_OUTCOME, VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;
          const [error] = JSON.parse(VALIDATION_ERRORS);

          assert.equal(STEP_OUTCOME, 'failure');
          assert.equal(VALIDATION_PASSED, 'false');
          assert.equal(VALIDATION_ERROR_COUNT, '1');

          assert.deepEqual(error, {
            kind: 'parse-error',
            file: 'plugins-list.yaml',
            message: 'plugins-list.yaml must be a YAML dictionary with plugin paths as keys'
          });

  test-validation-incompatible-workspace-skips-tag:
    name: Test Validation (incompatible workspace skips OCI tag check)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata with next__ computed prefix
        id: validate
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/next-prefix-skip
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}
          computed-image-tag-prefix: next__

      - name: Verify validation passed despite stale OCI tag
        env:
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;

          // Metadata has bs_1.40.0__1.0.0 which doesn't match target 1.42.5,
          // but with next__ computed prefix the OCI tag check is skipped
          assert.equal(VALIDATION_PASSED, 'true');
          assert.equal(VALIDATION_ERROR_COUNT, '0');
          assert.equal(VALIDATION_ERRORS, '[]');

  test-validation-pr-compatible-validates-tag:
    name: Test Validation (PR build with compatible workspace validates OCI tag)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata with bs_ computed prefix (simulating compatible PR)
        id: validate
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/pass
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}
          computed-image-tag-prefix: bs_${{ env.TARGET_BACKSTAGE_VERSION }}__

      - name: Verify validation passed (metadata tag matches computed prefix)
        env:
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;

          // Even though a real PR build would use pr_N__ as the image-tag-prefix,
          // the computed-image-tag-prefix is bs_<target>__ for compatible workspaces,
          // so OCI tag validation runs and passes against the production tag.
          assert.equal(VALIDATION_PASSED, 'true');
          assert.equal(VALIDATION_ERROR_COUNT, '0');
          assert.equal(VALIDATION_ERRORS, '[]');

  test-validation-pr-incompatible-skips-tag:
    name: Test Validation (PR build with incompatible workspace skips OCI tag check)
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Node.js 24.x
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        with:
          node-version: 24.x

      - name: Run Validate Metadata with next__ computed prefix (simulating incompatible PR)
        id: validate
        uses: ./validate-metadata
        with:
          overlay-root: ${{ env.TEST_DIR }}/cases/next-prefix-skip
          plugins-root: ${{ env.TEST_DIR }}/source
          target-backstage-version: ${{ env.TARGET_BACKSTAGE_VERSION }}
          image-repository-prefix: ${{ env.IMAGE_REPOSITORY_PREFIX }}
          computed-image-tag-prefix: next__

      - name: Verify validation passed despite stale OCI tag
        env:
          VALIDATION_PASSED: ${{ steps.validate.outputs.validation-passed }}
          VALIDATION_ERROR_COUNT: ${{ steps.validate.outputs.validation-error-count }}
          VALIDATION_ERRORS: ${{ steps.validate.outputs.validation-errors }}
        shell: node {0}
        run: |
          import assert from 'node:assert/strict';
          const { VALIDATION_PASSED, VALIDATION_ERROR_COUNT, VALIDATION_ERRORS } = process.env;

          // PR build of an incompatible workspace: computed prefix is next__,
          // so OCI tag validation is skipped even though metadata has stale bs_1.40.0__
          assert.equal(VALIDATION_PASSED, 'true');
          assert.equal(VALIDATION_ERROR_COUNT, '0');
          assert.equal(VALIDATION_ERRORS, '[]');