diff --git a/docs/user/reference/cli/azldev_image_test.md b/docs/user/reference/cli/azldev_image_test.md index 89c29c12..1d4bf3f5 100644 --- a/docs/user/reference/cli/azldev_image_test.md +++ b/docs/user/reference/cli/azldev_image_test.md @@ -11,7 +11,7 @@ project configuration. Test suites are defined in the [test-suites] section of azldev.toml and referenced by images via the [images.NAME.tests] subtable. Each test suite specifies a type -and framework-specific configuration in a matching subtable. +(pytest or lisa) and framework-specific configuration in a matching subtable. By default, all test suites associated with the named image are run. Use --test-suite to select specific suites (may be repeated). @@ -25,6 +25,12 @@ with the configured test paths and extra arguments. Use {image-path} in extra-args to insert the image path. Glob patterns (including **) in test-paths are expanded automatically. +For LISA tests, the test runner executes on the host and boots the image in a +QEMU VM. azldev clones the LISA framework, generates a runbook from the suite's +configured test cases, and runs it against the image. azldev generates an +ephemeral SSH key pair to access the booted VM and removes it once the suite +finishes. + ``` azldev image test IMAGE_NAME [flags] ``` diff --git a/internal/app/azldev/cmds/image/lisarunbook.go b/internal/app/azldev/cmds/image/lisarunbook.go new file mode 100644 index 00000000..bb543811 --- /dev/null +++ b/internal/app/azldev/cmds/image/lisarunbook.go @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package image + +import ( + "fmt" + "strings" + + "gopkg.in/yaml.v3" +) + +// The following structs model the subset of a LISA runbook that azldev generates. The +// YAML field names are dictated by LISA's runbook schema (snake_case), which we do not +// control. +// + +type lisaRunbook struct { + Name string `yaml:"name"` + Include []lisaInclude `yaml:"include"` + Testcase []lisaTestcase `yaml:"testcase"` + Notifier []lisaNotifier `yaml:"notifier"` + Platform []lisaPlatform `yaml:"platform"` +} + +type lisaInclude struct { + Path string `yaml:"path"` +} + +type lisaTestcase struct { + Criteria lisaCriteria `yaml:"criteria"` +} + +type lisaCriteria struct { + Name string `yaml:"name"` +} + +type lisaNotifier struct { + Type string `yaml:"type"` +} + +//nolint:tagliatelle // External schema (LISA runbook) dictates the field names. +type lisaPlatform struct { + Type string `yaml:"type"` + AdminPrivateKeyFile string `yaml:"admin_private_key_file"` + KeepEnvironment string `yaml:"keep_environment"` + Qemu lisaPlatformQemu `yaml:"qemu"` + Requirement lisaPlatformReqRoot `yaml:"requirement"` +} + +//nolint:tagliatelle // External schema (LISA runbook) dictates the field names. +type lisaPlatformQemu struct { + NetworkBootTimeout int `yaml:"network_boot_timeout"` +} + +type lisaPlatformReqRoot struct { + Qemu lisaPlatformReqQemu `yaml:"qemu"` +} + +type lisaPlatformReqQemu struct { + Qcow2 string `yaml:"qcow2"` +} + +const ( + // runbookTierIncludePath is the path (relative to the generated runbook) to the shared + // tier definitions in the LISA tree. LISA resolves includes relative to the runbook file's + // directory, so this resolves correctly only when the generated runbook is written at the + // framework repo root (see writeGeneratedRunbook). + runbookTierIncludePath = "lisa/microsoft/runbook/tiers/tier.yml" + // runbookBootTimeoutSeconds is the QEMU network boot timeout used in the generated runbook. + runbookBootTimeoutSeconds = 300 +) + +// generateRunbookYAML builds a LISA runbook that runs the given test cases on a QEMU VM +// booted from imagePath, authenticating with adminKeyPath. All values (image path, admin +// key path) are inlined as concrete values. keep_environment is "no" so LISA tears down the +// VM environment after the run. +func generateRunbookYAML(suiteName string, testCases []string, imagePath, adminKeyPath string) ([]byte, error) { + runbook := lisaRunbook{ + Name: suiteName, + Include: []lisaInclude{{Path: runbookTierIncludePath}}, + Testcase: []lisaTestcase{ + {Criteria: lisaCriteria{Name: strings.Join(testCases, "|")}}, + }, + Notifier: []lisaNotifier{{Type: "html"}}, + Platform: []lisaPlatform{ + { + Type: "qemu", + AdminPrivateKeyFile: adminKeyPath, + KeepEnvironment: "no", + Qemu: lisaPlatformQemu{NetworkBootTimeout: runbookBootTimeoutSeconds}, + Requirement: lisaPlatformReqRoot{ + Qemu: lisaPlatformReqQemu{ + Qcow2: imagePath, + }, + }, + }, + }, + } + + data, err := yaml.Marshal(&runbook) + if err != nil { + return nil, fmt.Errorf("failed to marshal generated LISA runbook:\n%w", err) + } + + return data, nil +} diff --git a/internal/app/azldev/cmds/image/lisarunbook_internal_test.go b/internal/app/azldev/cmds/image/lisarunbook_internal_test.go new file mode 100644 index 00000000..146e1445 --- /dev/null +++ b/internal/app/azldev/cmds/image/lisarunbook_internal_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package image + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" +) + +func TestGenerateRunbookYAML(t *testing.T) { + data, err := generateRunbookYAML( + "lisa-qemu", + []string{"verify_cpu_count", "verify_grub"}, + "/abs/image.qcow2", + "/home/user/.ssh/id_rsa", + ) + require.NoError(t, err) + + var runbook lisaRunbook + require.NoError(t, yaml.Unmarshal(data, &runbook)) + + assert.Equal(t, "lisa-qemu", runbook.Name) + require.Len(t, runbook.Include, 1) + assert.Equal(t, "lisa/microsoft/runbook/tiers/tier.yml", runbook.Include[0].Path) + + require.Len(t, runbook.Testcase, 1) + assert.Equal(t, "verify_cpu_count|verify_grub", runbook.Testcase[0].Criteria.Name) + + require.Len(t, runbook.Notifier, 1) + assert.Equal(t, "html", runbook.Notifier[0].Type) + + require.Len(t, runbook.Platform, 1) + platform := runbook.Platform[0] + assert.Equal(t, "qemu", platform.Type) + assert.Equal(t, "/home/user/.ssh/id_rsa", platform.AdminPrivateKeyFile) + assert.Equal(t, "no", platform.KeepEnvironment) + assert.Equal(t, runbookBootTimeoutSeconds, platform.Qemu.NetworkBootTimeout) + assert.Equal(t, "/abs/image.qcow2", platform.Requirement.Qemu.Qcow2) +} + +func TestGenerateRunbookYAML_SingleTestCase(t *testing.T) { + data, err := generateRunbookYAML("solo", []string{"verify_grub"}, "img", "key") + require.NoError(t, err) + + var runbook lisaRunbook + require.NoError(t, yaml.Unmarshal(data, &runbook)) + + require.Len(t, runbook.Testcase, 1) + assert.Equal(t, "verify_grub", runbook.Testcase[0].Criteria.Name) +} diff --git a/internal/app/azldev/cmds/image/lisarunner.go b/internal/app/azldev/cmds/image/lisarunner.go new file mode 100644 index 00000000..5da40d36 --- /dev/null +++ b/internal/app/azldev/cmds/image/lisarunner.go @@ -0,0 +1,399 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package image + +import ( + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/git" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" +) + +const ( + // lisaDirName is the parent directory under the project work dir for LISA-related state. + lisaDirName = "lisa" + // lisaVenvDirName is the name of the venv subdirectory for LISA. + lisaVenvDirName = "venv" + // lisaFrameworkDirName is the subdirectory for cloned LISA framework repos. + lisaFrameworkDirName = "framework" + // lisaProgram is the LISA executable name inside the venv. + lisaProgram = "lisa" + // shortSHALength is the number of characters to use from a SHA for directory names. + shortSHALength = 12 + // lisaGeneratedRunbookPrefix is prepended to generated runbook filenames. + lisaGeneratedRunbookPrefix = "azldev-generated-" +) + +// RunLisaSuite runs a LISA-based test suite by cloning the framework repo, setting up a +// venv, generating a runbook from the configured test cases, and invoking LISA. +func RunLisaSuite( + env *azldev.Env, suiteConfig *projectconfig.TestSuiteConfig, + imageConfig *projectconfig.ImageConfig, options *ImageTestOptions, +) error { + lisaConfig := suiteConfig.Lisa + if lisaConfig == nil { + return fmt.Errorf("test suite %#q is missing lisa configuration", suiteConfig.Name) + } + + slog.Info("Running LISA test suite", + slog.String("name", suiteConfig.Name), + slog.String("framework-ref", lisaConfig.Framework.Ref), + slog.Int("test-cases", len(lisaConfig.TestCases)), + slog.String("image-path", options.ImagePath), + ) + + // Ensure python3 and git are available. + if err := prereqs.RequireExecutable(env, pythonProgram, nil); err != nil { + return fmt.Errorf("python3 is required to run LISA tests:\n%w", err) + } + + if err := prereqs.RequireExecutable(env, "git", nil); err != nil { + return fmt.Errorf("git is required to clone LISA repos:\n%w", err) + } + + lisaBaseDir := filepath.Join(env.WorkDir(), lisaDirName) + + // Generate an ephemeral admin SSH key pair for VM access; it is removed once the + // suite finishes. + adminKeyPath, keyCleanup, err := generateAdminKeyPair(env, lisaBaseDir) + if err != nil { + return err + } + + defer keyCleanup() + + // Clone/update the LISA framework. + frameworkDir, err := ensureGitRepo( + env, lisaBaseDir, lisaFrameworkDirName, &lisaConfig.Framework, + ) + if err != nil { + return fmt.Errorf("failed to set up LISA framework:\n%w", err) + } + + // Set up or reuse the LISA venv and install the framework. + venvDir, err := ensureLisaVenv(env, suiteConfig.Name, frameworkDir, lisaConfig.PipPreInstall, lisaConfig.PipExtras) + if err != nil { + return err + } + + // The generated runbook boots a qcow2 disk on the qemu platform, so the image must + // already be in qcow2 format; any other format is rejected. + if err := requireQcow2Image(options.ImagePath); err != nil { + return err + } + + // Generate a runbook from the configured test cases and write it into the framework + // tree so its relative includes resolve. + runbookPath, err := writeGeneratedRunbook( + env, frameworkDir, suiteConfig.Name, lisaConfig.TestCases, options.ImagePath, adminKeyPath, + ) + if err != nil { + return err + } + + // Remove the generated runbook from the framework checkout once the suite finishes so + // stale azldev-generated-* files don't accumulate or influence future runs. + defer func() { + if removeErr := env.FS().RemoveAll(runbookPath); removeErr != nil { + slog.Warn("Failed to remove generated LISA runbook", + slog.String("path", runbookPath), slog.Any("error", removeErr)) + } + }() + + // Build LISA arguments with placeholder expansion. + lisaArgs := buildLisaArgs(runbookPath, lisaConfig.ExtraArgs, imageConfig, options) + + return runLisaCommand(env, venvDir, frameworkDir, lisaArgs) +} + +// runLisaCommand invokes the LISA executable from the framework's venv with the given args, +// streaming its output. LISA is run from the framework's lisa/ directory so relative +// extension paths (e.g., microsoft/testsuites) resolve correctly. +func runLisaCommand(env *azldev.Env, venvDir, frameworkDir string, lisaArgs []string) error { + slog.Info("Running LISA", slog.Any("args", lisaArgs)) + + lisaBin := filepath.Join(venvDir, "bin", lisaProgram) + lisaWorkDir := filepath.Join(frameworkDir, "lisa") + + lisaCmd := exec.CommandContext(env, lisaBin, lisaArgs...) + lisaCmd.Dir = lisaWorkDir + lisaCmd.Stdout = os.Stdout + lisaCmd.Stderr = os.Stderr + + cmd, err := env.Command(lisaCmd) + if err != nil { + return fmt.Errorf("failed to create LISA command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return fmt.Errorf("LISA test run failed:\n%w", err) + } + + return nil +} + +// writeGeneratedRunbook generates a LISA runbook from the given test cases and writes it at +// the framework repo root, so that its repo-root-relative tier include +// ('lisa/microsoft/runbook/tiers/tier.yml') resolves against the real tier definitions. +// It returns the absolute path to the written runbook. +func writeGeneratedRunbook( + env *azldev.Env, frameworkDir, suiteName string, testCases []string, + imagePath, adminKeyPath string, +) (string, error) { + if len(testCases) == 0 { + return "", fmt.Errorf("test suite %#q has no lisa test-cases to run", suiteName) + } + + absImagePath, err := filepath.Abs(imagePath) + if err != nil { + absImagePath = imagePath + } + + runbookYAML, err := generateRunbookYAML(suiteName, testCases, absImagePath, adminKeyPath) + if err != nil { + return "", err + } + + // Write the runbook at the framework repo root so its repo-root-relative include resolves. + runbookPath := filepath.Join(frameworkDir, lisaGeneratedRunbookPrefix+suiteName+".yml") + + if err := fileutils.WriteFile(env.FS(), runbookPath, runbookYAML, fileperms.PrivateFile); err != nil { + return "", fmt.Errorf("failed to write generated runbook %#q:\n%w", runbookPath, err) + } + + slog.Info("Generated LISA runbook", slog.String("path", runbookPath)) + + return runbookPath, nil +} + +// requireQcow2Image verifies that the given disk image is in qcow2 format, which the +// generated LISA runbook requires (LISA's qemu platform boots qcow2 disks). Any other +// format is rejected. +func requireQcow2Image(imagePath string) error { + format, err := InferImageFormat(imagePath) + if err != nil { + return err + } + + if format != string(ImageFormatQcow2) { + return fmt.Errorf( + "unsupported image format %#q for %#q: LISA requires a qcow2 image", + format, imagePath, + ) + } + + return nil +} + +// generateAdminKeyPair generates an ephemeral RSA SSH key pair under lisaBaseDir for LISA to +// use when accessing the booted VM. It returns the absolute path to the private key and a +// cleanup function that deletes the key pair once the suite finishes. +func generateAdminKeyPair( + env *azldev.Env, lisaBaseDir string, +) (keyPath string, cleanup func(), err error) { + if err := prereqs.RequireExecutable(env, "ssh-keygen", nil); err != nil { + return "", nil, fmt.Errorf("ssh-keygen is required to generate an admin SSH key:\n%w", err) + } + + if err := fileutils.MkdirAll(env.FS(), lisaBaseDir); err != nil { + return "", nil, fmt.Errorf("failed to create LISA base dir %#q:\n%w", lisaBaseDir, err) + } + + keysDir, err := fileutils.MkdirTemp(env.FS(), lisaBaseDir, "admin-key-*") + if err != nil { + return "", nil, fmt.Errorf("failed to create temporary admin key dir:\n%w", err) + } + + privateKeyPath := filepath.Join(keysDir, "id_rsa") + + slog.Info("Generating ephemeral admin SSH key pair", slog.String("path", privateKeyPath)) + + keygenCmd := exec.CommandContext( + env, "ssh-keygen", "-t", "rsa", "-b", "4096", "-f", privateKeyPath, "-N", "", "-q", + ) + + cmd, err := env.Command(keygenCmd) + if err != nil { + return "", nil, fmt.Errorf("failed to create ssh-keygen command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return "", nil, fmt.Errorf("failed to generate admin SSH key pair:\n%w", err) + } + + cleanup = func() { + slog.Info("Removing ephemeral admin SSH key pair", slog.String("path", keysDir)) + + if removeErr := env.FS().RemoveAll(keysDir); removeErr != nil { + slog.Warn("Failed to remove ephemeral admin SSH key pair", + slog.String("path", keysDir), slog.Any("error", removeErr)) + } + } + + return privateKeyPath, cleanup, nil +} + +// ensureGitRepo clones a git repo (if not already present) and checks out the specified +// commit SHA. Returns the path to the cloned repo directory. +func ensureGitRepo( + env *azldev.Env, baseDir string, category string, source *projectconfig.GitSourceConfig, +) (string, error) { + if len(source.Ref) < shortSHALength { + return "", fmt.Errorf( + "invalid git ref %#q: must be at least %d characters", source.Ref, shortSHALength, + ) + } + + shortSHA := source.Ref[:shortSHALength] + repoDir := filepath.Join(baseDir, category, shortSHA) + + repoExists, err := fileutils.DirExists(env.FS(), repoDir) + if err != nil { + return "", fmt.Errorf("cannot check repo dir at %#q:\n%w", repoDir, err) + } + + gitProvider, err := git.NewGitProviderImpl(env, env) + if err != nil { + return "", fmt.Errorf("failed to create git provider:\n%w", err) + } + + if !repoExists { + slog.Info("Cloning git repo", + slog.String("url", source.GitURL), + slog.String("ref", source.Ref), + slog.String("dest", repoDir), + ) + + if err := gitProvider.Clone(env, source.GitURL, repoDir); err != nil { + return "", fmt.Errorf("failed to clone %#q:\n%w", source.GitURL, err) + } + } else { + slog.Info("Reusing existing git repo", + slog.String("path", repoDir), + slog.String("ref", source.Ref), + ) + } + + // Always checkout the pinned ref, even when reusing an existing checkout, so an + // interrupted or externally-modified clone still runs against the correct revision. + if err := gitProvider.Checkout(env, repoDir, source.Ref); err != nil { + return "", fmt.Errorf("failed to checkout %#q:\n%w", source.Ref, err) + } + + return repoDir, nil +} + +// ensureLisaVenv creates or reuses a Python venv for LISA and installs the framework via +// pip install -e. If pipExtras are specified, they are appended as pip extras +// (e.g., pip install -e ".[azure,legacy]"). +func ensureLisaVenv( + env *azldev.Env, suiteName string, frameworkDir string, + pipPreInstall []string, pipExtras []string, +) (string, error) { + venvDir := filepath.Join(env.WorkDir(), lisaDirName, lisaVenvDirName, suiteName) + + venvPython := filepath.Join(venvDir, "bin", pythonProgram) + + venvExists, err := fileutils.Exists(env.FS(), venvPython) + if err != nil { + return "", fmt.Errorf("cannot check LISA venv at %#q:\n%w", venvDir, err) + } + + if !venvExists { + slog.Info("Creating LISA Python venv", slog.String("path", venvDir)) + + if err := createPythonVenv(env, venvDir); err != nil { + return "", err + } + } else { + slog.Info("Reusing existing LISA venv", slog.String("path", venvDir)) + } + + // Install pre-install packages before the framework (to override version pins). + if len(pipPreInstall) > 0 { + slog.Info("Installing pre-install packages", slog.Any("packages", pipPreInstall)) + + preInstallArgs := append([]string{"-m", "pip", "install", "--quiet"}, pipPreInstall...) + + preInstallCmd := exec.CommandContext(env, venvPython, preInstallArgs...) + preInstallCmd.Stdout = os.Stdout + preInstallCmd.Stderr = os.Stderr + + cmd, err := env.Command(preInstallCmd) + if err != nil { + return "", fmt.Errorf("failed to create pip pre-install command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return "", fmt.Errorf("failed to install pre-install packages:\n%w", err) + } + } + + // Always refresh LISA installation from the framework directory. + slog.Info("Installing LISA framework", + slog.String("framework", frameworkDir), + ) + + // Build the pip install target, appending extras if specified. + pipTarget := frameworkDir + if len(pipExtras) > 0 { + pipTarget = frameworkDir + "[" + strings.Join(pipExtras, ",") + "]" + } + + pipCmd := exec.CommandContext( + env, venvPython, "-m", "pip", "install", "--quiet", "-e", pipTarget, + ) + pipCmd.Stdout = os.Stdout + pipCmd.Stderr = os.Stderr + + cmd, err := env.Command(pipCmd) + if err != nil { + return "", fmt.Errorf("failed to create pip install command:\n%w", err) + } + + if err := cmd.Run(env); err != nil { + return "", fmt.Errorf("failed to install LISA framework:\n%w", err) + } + + return venvDir, nil +} + +// buildLisaArgs constructs the LISA command-line arguments. The runbook path is passed +// via -r, and extra-args are appended after placeholder expansion. +func buildLisaArgs( + runbookPath string, + extraArgs []string, + imageConfig *projectconfig.ImageConfig, + options *ImageTestOptions, +) []string { + absImagePath, err := filepath.Abs(options.ImagePath) + if err != nil { + absImagePath = options.ImagePath + } + + replacer := strings.NewReplacer( + imagePlaceholder, absImagePath, + imageNamePlaceholder, options.ImageName, + capabilitiesPlaceholder, strings.Join(imageConfig.Capabilities.EnabledNames(), ","), + ) + + baseArgs := []string{"-r", runbookPath} + args := make([]string, 0, len(baseArgs)+len(extraArgs)) + args = append(args, baseArgs...) + + for _, arg := range extraArgs { + args = append(args, replacer.Replace(arg)) + } + + return args +} diff --git a/internal/app/azldev/cmds/image/test.go b/internal/app/azldev/cmds/image/test.go index 9ea9ad54..dee998fd 100644 --- a/internal/app/azldev/cmds/image/test.go +++ b/internal/app/azldev/cmds/image/test.go @@ -54,7 +54,7 @@ project configuration. Test suites are defined in the [test-suites] section of azldev.toml and referenced by images via the [images.NAME.tests] subtable. Each test suite specifies a type -and framework-specific configuration in a matching subtable. +(pytest or lisa) and framework-specific configuration in a matching subtable. By default, all test suites associated with the named image are run. Use --test-suite to select specific suites (may be repeated). @@ -66,7 +66,13 @@ For pytest tests, azldev creates a Python virtual environment, installs dependencies from pyproject.toml in the working directory, and runs pytest with the configured test paths and extra arguments. Use {image-path} in extra-args to insert the image path. Glob patterns (including **) in -test-paths are expanded automatically.`, +test-paths are expanded automatically. + +For LISA tests, the test runner executes on the host and boots the image in a +QEMU VM. azldev clones the LISA framework, generates a runbook from the suite's +configured test cases, and runs it against the image. azldev generates an +ephemeral SSH key pair to access the booted VM and removes it once the suite +finishes.`, Example: ` # Run all test suites for an image (artifact auto-resolved from output dir) azldev image test vm-base @@ -258,8 +264,7 @@ func runTestSuite( return RunPytestSuite(env, suiteConfig, imageConfig, options) case projectconfig.TestTypeLisa: - return fmt.Errorf("LISA test suites cannot be run locally via 'azldev image test'; "+ - "test suite %#q must be run through the LISA infrastructure", suiteConfig.Name) + return RunLisaSuite(env, suiteConfig, imageConfig, options) default: return fmt.Errorf("unsupported test type %#q for test suite %#q", suiteConfig.Type, suiteConfig.Name) diff --git a/internal/projectconfig/loader_test.go b/internal/projectconfig/loader_test.go index 67d781ce..d362040d 100644 --- a/internal/projectconfig/loader_test.go +++ b/internal/projectconfig/loader_test.go @@ -882,6 +882,18 @@ description = "Smoke tests for images" working-dir = "tests" test-paths = ["cases/test_*.py"] extra-args = ["--image-path", "{image-path}"] + +[test-suites.integration] +type = "lisa" +description = "LISA integration tests" + +[test-suites.integration.lisa] +test-cases = ["verify_cpu_count", "verify_grub"] +extra-args = ["-v", "qcow2:{image-path}"] + +[test-suites.integration.lisa.framework] +git-url = "https://github.com/microsoft/lisa.git" +ref = "abcdef0123456789abcdef0123456789abcdef01" ` configDir := filepath.Dir(testConfigPath) @@ -892,7 +904,7 @@ extra-args = ["--image-path", "{image-path}"] config, err := loadAndResolveProjectConfig(ctx.FS(), false, testConfigPath) require.NoError(t, err) - require.Len(t, config.TestSuites, 1) + require.Len(t, config.TestSuites, 2) // Check pytest test. if assert.Contains(t, config.TestSuites, "smoke") { @@ -905,6 +917,19 @@ extra-args = ["--image-path", "{image-path}"] assert.Equal(t, []string{"cases/test_*.py"}, smokeTest.Pytest.TestPaths) assert.Equal(t, []string{"--image-path", "{image-path}"}, smokeTest.Pytest.ExtraArgs) } + + // Check LISA test. + if assert.Contains(t, config.TestSuites, "integration") { + lisaTest := config.TestSuites["integration"] + assert.Equal(t, "integration", lisaTest.Name) + assert.Equal(t, TestTypeLisa, lisaTest.Type) + assert.Equal(t, "LISA integration tests", lisaTest.Description) + require.NotNil(t, lisaTest.Lisa) + assert.Equal(t, "https://github.com/microsoft/lisa.git", lisaTest.Lisa.Framework.GitURL) + assert.Equal(t, "abcdef0123456789abcdef0123456789abcdef01", lisaTest.Lisa.Framework.Ref) + assert.Equal(t, []string{"verify_cpu_count", "verify_grub"}, lisaTest.Lisa.TestCases) + assert.Equal(t, []string{"-v", "qcow2:{image-path}"}, lisaTest.Lisa.ExtraArgs) + } } func TestLoadAndResolveProjectConfig_DuplicateTests(t *testing.T) { diff --git a/internal/projectconfig/testsuite.go b/internal/projectconfig/testsuite.go index d8424592..7a518932 100644 --- a/internal/projectconfig/testsuite.go +++ b/internal/projectconfig/testsuite.go @@ -4,6 +4,7 @@ package projectconfig import ( + "encoding/hex" "errors" "fmt" @@ -16,7 +17,7 @@ type TestType string const ( // TestTypePytest uses pytest to run static/offline validation checks. TestTypePytest TestType = "pytest" - // TestTypeLisa uses LISA (Linux Integration Services Automation) to run VM-level tests. + // TestTypeLisa uses the LISA framework to run live VM tests. TestTypeLisa TestType = "lisa" ) @@ -34,6 +35,8 @@ var ( ErrMismatchedTestSubtable = errors.New("mismatched test subtable") // ErrInvalidInstallMode is returned when a [PytestConfig.Install] value is not recognized. ErrInvalidInstallMode = errors.New("invalid install mode") + // ErrInvalidGitRef is returned when a git ref is not a valid hex commit SHA. + ErrInvalidGitRef = errors.New("invalid git ref") ) // TestSuiteConfig defines a named test suite. @@ -50,6 +53,10 @@ type TestSuiteConfig struct { // Pytest holds pytest-specific configuration. Required when Type is "pytest". Pytest *PytestConfig `toml:"pytest,omitempty" json:"pytest,omitempty" jsonschema:"title=Pytest config,description=Pytest-specific configuration (required when type is pytest)"` + // Lisa holds LISA-specific configuration. Optional for a "lisa" suite; when present + // it drives generation and execution of a LISA runbook. + Lisa *LisaConfig `toml:"lisa,omitempty" json:"lisa,omitempty" jsonschema:"title=LISA config,description=LISA-specific configuration used to generate and run a LISA runbook (optional)"` + // Reference to the source config file that this definition came from; not present // in serialized files. SourceConfigFile *ConfigFile `toml:"-" json:"-" table:"-"` @@ -91,6 +98,73 @@ type PytestConfig struct { Install PytestInstallMode `toml:"install,omitempty" json:"install,omitempty" jsonschema:"enum=pyproject,enum=requirements,enum=none,title=Install mode,description=How to install Python dependencies: pyproject\\, requirements\\, or none (default)"` } +// LisaConfig holds configuration specific to LISA-based test suites. +type LisaConfig struct { + // Framework identifies the git source for the LISA framework itself. + Framework GitSourceConfig `toml:"framework" json:"framework" jsonschema:"required,title=Framework,description=Git source for the LISA framework"` + + // TestCases lists the LISA test case names to run. They are combined (joined with '|') + // into the criteria of a runbook that azldev generates and runs. Required when a [lisa] + // subtable is present. + TestCases []string `toml:"test-cases" json:"testCases" jsonschema:"required,title=Test cases,description=LISA test case names to run; combined into a generated runbook's criteria"` + + // PipPreInstall lists pip packages to install before the LISA framework itself. + // This can be used to override framework version pins that conflict with the local + // environment (e.g., installing a system-matching libvirt-python version). + PipPreInstall []string `toml:"pip-pre-install,omitempty" json:"pipPreInstall,omitempty" jsonschema:"title=Pip pre-install,description=Pip packages to install before the framework (for overriding version pins)"` + + // PipExtras lists pip extras to install from the LISA framework package (e.g., "azure", + // "legacy"). These are appended to the pip install command as pip install -e ".[extra1,extra2]". + PipExtras []string `toml:"pip-extras,omitempty" json:"pipExtras,omitempty" jsonschema:"title=Pip extras,description=Pip extras to install from the LISA framework package"` + + // ExtraArgs is the list of additional arguments to pass to LISA. These are passed + // verbatim after placeholder substitution. Supports {image-path}, {image-name}, + // and {capabilities} placeholders. + ExtraArgs []string `toml:"extra-args,omitempty" json:"extraArgs,omitempty" jsonschema:"title=Extra arguments,description=Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders."` +} + +// GitSourceConfig identifies a git repository at a specific commit. +type GitSourceConfig struct { + // GitURL is the URL of the git repository. + GitURL string `toml:"git-url" json:"gitUrl" jsonschema:"required,title=Git URL,description=URL of the git repository"` + + // Ref is the commit SHA to check out. Must be a full hex commit hash. + Ref string `toml:"ref" json:"ref" jsonschema:"required,title=Ref,description=Commit SHA to check out (full hex hash)"` +} + +// Validate checks that the [GitSourceConfig] has required fields and a valid ref. +func (g *GitSourceConfig) Validate(context string) error { + if g.GitURL == "" { + return fmt.Errorf("%w: %s requires 'git-url'", ErrMissingTestField, context) + } + + if g.Ref == "" { + return fmt.Errorf("%w: %s requires 'ref'", ErrMissingTestField, context) + } + + if err := validateCommitSHA(g.Ref); err != nil { + return fmt.Errorf("%s: %w", context, err) + } + + return nil +} + +// validateCommitSHA checks that ref is a valid full-length hex commit SHA (40 characters). +func validateCommitSHA(ref string) error { + const commitSHALength = 40 + + if len(ref) != commitSHALength { + return fmt.Errorf("%w: expected %d hex characters, got %d: %#q", + ErrInvalidGitRef, commitSHALength, len(ref), ref) + } + + if _, err := hex.DecodeString(ref); err != nil { + return fmt.Errorf("%w: not a valid hex string: %#q", ErrInvalidGitRef, ref) + } + + return nil +} + // Validate checks that the test suite config has valid type-specific required fields and that // only the matching subtable is present. func (t *TestSuiteConfig) Validate() error { @@ -110,16 +184,28 @@ func (t *TestSuiteConfig) Validate() error { return fmt.Errorf("test suite %#q: %w", t.Name, err) } + if t.Lisa != nil { + return fmt.Errorf("%w: test suite %#q of type %#q must not have a [lisa] subtable", + ErrMismatchedTestSubtable, t.Name, t.Type) + } + case TestTypeLisa: - // LISA is an external test framework not executed by azldev. - // Suites of this type serve as metadata for external orchestration (e.g. control tower). + // The [lisa] subtable is optional; when present it must be internally consistent. + if t.Lisa != nil { + frameworkContext := fmt.Sprintf("test suite %#q lisa.framework", t.Name) + if err := t.Lisa.Framework.Validate(frameworkContext); err != nil { + return err + } + + if len(t.Lisa.TestCases) == 0 { + return fmt.Errorf("%w: test suite %#q lisa requires 'test-cases'", + ErrMissingTestField, t.Name) + } + } + if t.Pytest != nil { - return fmt.Errorf( - "%w: test suite %#q of type %#q cannot include subtable 'pytest'", - ErrMismatchedTestSubtable, - t.Name, - t.Type, - ) + return fmt.Errorf("%w: test suite %#q of type %#q must not have a [pytest] subtable", + ErrMismatchedTestSubtable, t.Name, t.Type) } default: @@ -202,5 +288,15 @@ func (t *TestSuiteConfig) WithAbsolutePaths(referenceDir string) *TestSuiteConfi } } + if t.Lisa != nil { + result.Lisa = &LisaConfig{ + Framework: t.Lisa.Framework, + TestCases: append([]string(nil), t.Lisa.TestCases...), + PipPreInstall: append([]string(nil), t.Lisa.PipPreInstall...), + PipExtras: append([]string(nil), t.Lisa.PipExtras...), + ExtraArgs: append([]string(nil), t.Lisa.ExtraArgs...), + } + } + return result } diff --git a/internal/projectconfig/testsuite_test.go b/internal/projectconfig/testsuite_test.go index e24f758f..de95adb6 100644 --- a/internal/projectconfig/testsuite_test.go +++ b/internal/projectconfig/testsuite_test.go @@ -12,6 +12,20 @@ import ( "github.com/stretchr/testify/require" ) +// validTestSHA is a 40-character hex string for use in tests. +const validTestSHA = "abcdef0123456789abcdef0123456789abcdef01" + +// validLisaConfig returns a valid [projectconfig.LisaConfig] for use in tests. +func validLisaConfig() *projectconfig.LisaConfig { + return &projectconfig.LisaConfig{ + Framework: projectconfig.GitSourceConfig{ + GitURL: "https://github.com/microsoft/lisa.git", + Ref: validTestSHA, + }, + TestCases: []string{"verify_cpu_count", "verify_grub"}, + } +} + func TestImageCapabilities_EnabledNames(t *testing.T) { t.Run("all enabled", func(t *testing.T) { caps := projectconfig.ImageCapabilities{ @@ -170,6 +184,15 @@ func TestTestSuiteConfig_Validate(t *testing.T) { assert.NoError(t, testConfig.Validate()) }) + t.Run("valid lisa config", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: validLisaConfig(), + } + assert.NoError(t, testConfig.Validate()) + }) + t.Run("pytest missing subtable", func(t *testing.T) { testConfig := projectconfig.TestSuiteConfig{ Name: "smoke", @@ -181,37 +204,77 @@ func TestTestSuiteConfig_Validate(t *testing.T) { assert.Contains(t, err.Error(), "[pytest]") }) - t.Run("valid lisa type", func(t *testing.T) { + t.Run("pytest with lisa subtable", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "smoke", + Type: projectconfig.TestTypePytest, + Pytest: &projectconfig.PytestConfig{}, + Lisa: validLisaConfig(), + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) + }) + + t.Run("lisa missing subtable is valid (optional)", func(t *testing.T) { testConfig := projectconfig.TestSuiteConfig{ - Name: "vm-tests", + Name: "integration", Type: projectconfig.TestTypeLisa, } assert.NoError(t, testConfig.Validate()) }) - t.Run("valid lisa type with description", func(t *testing.T) { + t.Run("lisa missing framework git-url", func(t *testing.T) { + cfg := validLisaConfig() + cfg.Framework.GitURL = "" testConfig := projectconfig.TestSuiteConfig{ - Name: "vm-tests", - Description: "VM integration tests using LISA", - Type: projectconfig.TestTypeLisa, + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: cfg, } - assert.NoError(t, testConfig.Validate()) + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "git-url") }) - t.Run("lisa rejects pytest subtable", func(t *testing.T) { + t.Run("lisa invalid framework ref", func(t *testing.T) { + cfg := validLisaConfig() + cfg.Framework.Ref = "not-a-sha" testConfig := projectconfig.TestSuiteConfig{ - Name: "vm-tests", + Name: "integration", Type: projectconfig.TestTypeLisa, - Pytest: &projectconfig.PytestConfig{ - WorkingDir: "tests", - }, + Lisa: cfg, } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrInvalidGitRef) + }) + t.Run("lisa missing test-cases", func(t *testing.T) { + cfg := validLisaConfig() + cfg.TestCases = nil + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: cfg, + } + err := testConfig.Validate() + require.Error(t, err) + require.ErrorIs(t, err, projectconfig.ErrMissingTestField) + assert.Contains(t, err.Error(), "test-cases") + }) + + t.Run("lisa with pytest subtable", func(t *testing.T) { + testConfig := projectconfig.TestSuiteConfig{ + Name: "integration", + Type: projectconfig.TestTypeLisa, + Lisa: validLisaConfig(), + Pytest: &projectconfig.PytestConfig{}, + } err := testConfig.Validate() require.Error(t, err) require.ErrorIs(t, err, projectconfig.ErrMismatchedTestSubtable) - assert.Contains(t, err.Error(), "vm-tests") - assert.Contains(t, err.Error(), string(projectconfig.TestTypeLisa)) }) t.Run("unknown test type", func(t *testing.T) { diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 0cfae46f..1aef65db 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -592,6 +592,26 @@ "additionalProperties": false, "type": "object" }, + "GitSourceConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref" + ] + }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -709,6 +729,53 @@ "additionalProperties": false, "type": "object" }, + "LisaConfig": { + "properties": { + "framework": { + "$ref": "#/$defs/GitSourceConfig", + "title": "Framework", + "description": "Git source for the LISA framework" + }, + "test-cases": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Test cases", + "description": "LISA test case names to run; combined into a generated runbook's criteria" + }, + "pip-pre-install": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip pre-install", + "description": "Pip packages to install before the framework (for overriding version pins)" + }, + "pip-extras": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip extras", + "description": "Pip extras to install from the LISA framework package" + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "framework", + "test-cases" + ] + }, "Origin": { "properties": { "type": { @@ -1332,6 +1399,11 @@ "$ref": "#/$defs/PytestConfig", "title": "Pytest config", "description": "Pytest-specific configuration (required when type is pytest)" + }, + "lisa": { + "$ref": "#/$defs/LisaConfig", + "title": "LISA config", + "description": "LISA-specific configuration used to generate and run a LISA runbook (optional)" } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 0cfae46f..1aef65db 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -592,6 +592,26 @@ "additionalProperties": false, "type": "object" }, + "GitSourceConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref" + ] + }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -709,6 +729,53 @@ "additionalProperties": false, "type": "object" }, + "LisaConfig": { + "properties": { + "framework": { + "$ref": "#/$defs/GitSourceConfig", + "title": "Framework", + "description": "Git source for the LISA framework" + }, + "test-cases": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Test cases", + "description": "LISA test case names to run; combined into a generated runbook's criteria" + }, + "pip-pre-install": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip pre-install", + "description": "Pip packages to install before the framework (for overriding version pins)" + }, + "pip-extras": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip extras", + "description": "Pip extras to install from the LISA framework package" + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "framework", + "test-cases" + ] + }, "Origin": { "properties": { "type": { @@ -1332,6 +1399,11 @@ "$ref": "#/$defs/PytestConfig", "title": "Pytest config", "description": "Pytest-specific configuration (required when type is pytest)" + }, + "lisa": { + "$ref": "#/$defs/LisaConfig", + "title": "LISA config", + "description": "LISA-specific configuration used to generate and run a LISA runbook (optional)" } }, "additionalProperties": false, diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 0cfae46f..1aef65db 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -592,6 +592,26 @@ "additionalProperties": false, "type": "object" }, + "GitSourceConfig": { + "properties": { + "git-url": { + "type": "string", + "title": "Git URL", + "description": "URL of the git repository" + }, + "ref": { + "type": "string", + "title": "Ref", + "description": "Commit SHA to check out (full hex hash)" + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "git-url", + "ref" + ] + }, "ImageCapabilities": { "properties": { "machine-bootable": { @@ -709,6 +729,53 @@ "additionalProperties": false, "type": "object" }, + "LisaConfig": { + "properties": { + "framework": { + "$ref": "#/$defs/GitSourceConfig", + "title": "Framework", + "description": "Git source for the LISA framework" + }, + "test-cases": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Test cases", + "description": "LISA test case names to run; combined into a generated runbook's criteria" + }, + "pip-pre-install": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip pre-install", + "description": "Pip packages to install before the framework (for overriding version pins)" + }, + "pip-extras": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Pip extras", + "description": "Pip extras to install from the LISA framework package" + }, + "extra-args": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Extra arguments", + "description": "Additional arguments passed to LISA. Supports {image-path} {image-name} {capabilities} placeholders." + } + }, + "additionalProperties": false, + "type": "object", + "required": [ + "framework", + "test-cases" + ] + }, "Origin": { "properties": { "type": { @@ -1332,6 +1399,11 @@ "$ref": "#/$defs/PytestConfig", "title": "Pytest config", "description": "Pytest-specific configuration (required when type is pytest)" + }, + "lisa": { + "$ref": "#/$defs/LisaConfig", + "title": "LISA config", + "description": "LISA-specific configuration used to generate and run a LISA runbook (optional)" } }, "additionalProperties": false,