tmt/tests: Add works_for_composefs meta field

Use the new `works_for_composefs` field to check whether a test is going
to work for a composefs booted system or not. This allows us to simply
do `cargo xtask test-tmt integration` and tests will be filtered by
variant

Signed-off-by: Pragyan Poudyal <pragyanpoudyal41999@gmail.com>

Author: Johan-Liebert1

Justfile

@@ -112,15 +112,7 @@ test-composefs-sealeduki-sdboot:
 [group('core')]
 test-composefs bootloader:
     just variant=composefs bootloader={{bootloader}} \
-        test-tmt --composefs-backend --bootloader {{bootloader}} \
-        readonly \
-        download-only \
-        image-pushpull-upgrade \
-        image-upgrade-reboot \
-        install-outside-container \
-        install-to-filesystem-var-mount \
-        soft-reboot \
-        usroverlay
+        test-tmt --composefs-backend --bootloader {{bootloader}} integration
 
 # Build and test composefs images booted using Type1 boot entries and systemd-boot as the bootloader
 [group('core')]

crates/xtask/src/tmt.rs

@@ -22,6 +22,8 @@ const FIELD_TRY_BIND_STORAGE: &str = "try_bind_storage";
 const FIELD_SUMMARY: &str = "summary";
 const FIELD_ADJUST: &str = "adjust";
 
+const FIELD_WORKS_FOR_COMPOSEFS: &str = "works_for_composefs";
+
 // bcvk options
 const BCVK_OPT_BIND_STORAGE_RO: &str = "--bind-storage-ro";
 const ENV_BOOTC_UPGRADE_IMAGE: &str = "BOOTC_upgrade_image";
@@ -247,9 +249,17 @@ fn verify_ssh_connectivity(sh: &Shell, port: u16, key_path: &Utf8Path) -> Result
     )
 }
 
+#[derive(Debug)]
+struct PlanMetadata {
+    try_bind_storage: bool,
+    works_for_composefs: bool,
+}
+
 /// Parse integration.fmf to extract extra-try_bind_storage for all plans
 #[context("Parsing integration.fmf")]
-fn parse_plan_metadata(plans_file: &Utf8Path) -> Result<std::collections::HashMap<String, bool>> {
+fn parse_plan_metadata(
+    plans_file: &Utf8Path,
+) -> Result<std::collections::HashMap<String, PlanMetadata>> {
     let content = std::fs::read_to_string(plans_file)?;
     let yaml = serde_yaml::from_str::<serde_yaml::Value>(&content)
         .context("Failed to parse integration.fmf YAML")?;
@@ -258,7 +268,8 @@ fn parse_plan_metadata(plans_file: &Utf8Path) -> Result<std::collections::HashMa
         anyhow::bail!("Expected YAML mapping in integration.fmf");
     };
 
-    let mut plan_metadata = std::collections::HashMap::new();
+    let mut plan_metadata: std::collections::HashMap<String, PlanMetadata> =
+        std::collections::HashMap::new();
 
     for (key, value) in mapping {
         let Some(plan_name) = key.as_str() else {
@@ -271,12 +282,34 @@ fn parse_plan_metadata(plans_file: &Utf8Path) -> Result<std::collections::HashMa
         let Some(plan_data) = value.as_mapping() else {
             continue;
         };
+
         if let Some(try_bind) = plan_data.get(&serde_yaml::Value::String(format!(
             "extra-{}",
             FIELD_TRY_BIND_STORAGE
         ))) {
             if let Some(b) = try_bind.as_bool() {
-                plan_metadata.insert(plan_name.to_string(), b);
+                plan_metadata
+                    .entry(plan_name.to_string())
+                    .and_modify(|m| m.try_bind_storage = b)
+                    .or_insert(PlanMetadata {
+                        try_bind_storage: b,
+                        works_for_composefs: false,
+                    });
+            }
+        }
+
+        if let Some(works_for_composefs) = plan_data.get(&serde_yaml::Value::String(format!(
+            "extra-{}",
+            FIELD_WORKS_FOR_COMPOSEFS
+        ))) {
+            if let Some(b) = works_for_composefs.as_bool() {
+                plan_metadata
+                    .entry(plan_name.to_string())
+                    .and_modify(|m| m.works_for_composefs = b)
+                    .or_insert(PlanMetadata {
+                        works_for_composefs: b,
+                        try_bind_storage: false,
+                    });
             }
         }
     }
@@ -359,18 +392,30 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
         .filter(|line| !line.is_empty() && line.starts_with("/"))
         .collect();
 
+    let original_plans_count = plans.len();
+
     // Filter plans based on user arguments
     if !filter_args.is_empty() {
-        let original_count = plans.len();
         plans.retain(|plan| filter_args.iter().any(|arg| plan.contains(arg.as_str())));
-        if plans.len() < original_count {
-            println!(
-                "Filtered from {} to {} plan(s) based on arguments: {:?}",
-                original_count,
-                plans.len(),
-                filter_args
-            );
-        }
+    }
+
+    if args.composefs_backend {
+        plans.retain(|plan| {
+            plan_metadata
+                .iter()
+                .find(|(key, _)| plan.ends_with(key.as_str()))
+                .map(|(_, v)| v.works_for_composefs)
+                .unwrap_or(false)
+        });
+    }
+
+    if plans.len() < original_plans_count {
+        println!(
+            "Filtered from {} to {} plan(s) based on arguments: {:?}",
+            original_plans_count,
+            plans.len(),
+            filter_args
+        );
     }
 
     if plans.is_empty() {
@@ -412,7 +457,7 @@ pub(crate) fn run_tmt(sh: &Shell, args: &RunTmtArgs) -> Result<()> {
             let try_bind_storage = plan_metadata
                 .iter()
                 .find(|(key, _)| plan.ends_with(key.as_str()))
-                .map(|(_, &v)| v)
+                .map(|(_, v)| v.try_bind_storage)
                 .unwrap_or(false);
 
             let mut opts = Vec::new();
@@ -863,6 +908,8 @@ struct TestDef {
     test_command: String,
     /// Whether this test wants to try bind storage (if distro supports it)
     try_bind_storage: bool,
+    /// Whether this test will work for composefs backend
+    works_for_composefs: bool,
     /// TMT fmf attributes to pass through (summary, duration, adjust, etc.)
     tmt: serde_yaml::Value,
 }
@@ -953,11 +1000,23 @@ pub(crate) fn update_integration() -> Result<()> {
             .and_then(|v| v.as_bool())
             .unwrap_or(false);
 
+        let works_for_composefs = metadata
+            .extra
+            .as_mapping()
+            .and_then(|m| {
+                m.get(&serde_yaml::Value::String(
+                    FIELD_WORKS_FOR_COMPOSEFS.to_string(),
+                ))
+            })
+            .and_then(|v| v.as_bool())
+            .unwrap_or(false);
+
         tests.push(TestDef {
             number: metadata.number,
             name: display_name,
             test_command,
             try_bind_storage,
+            works_for_composefs,
             tmt: metadata.tmt,
         });
     }
@@ -1076,6 +1135,13 @@ pub(crate) fn update_integration() -> Result<()> {
             );
         }
 
+        if test.works_for_composefs {
+            plan_value.insert(
+                serde_yaml::Value::String(format!("extra-{}", FIELD_WORKS_FOR_COMPOSEFS)),
+                serde_yaml::Value::Bool(true),
+            );
+        }
+
         plans_mapping.insert(
             serde_yaml::Value::String(plan_key),
             serde_yaml::Value::Mapping(plan_value),

tmt/plans/integration.fmf

@@ -49,13 +49,15 @@ execute:
     test:
       - /tmt/tests/tests/test-01-readonly
   extra-try_bind_storage: true
+  extra-works_for_composefs: true
 
 /plan-20-image-pushpull-upgrade:
   summary: Execute local upgrade tests
   discover:
     how: fmf
     test:
       - /tmt/tests/tests/test-20-image-pushpull-upgrade
+  extra-works_for_composefs: true
 
 /plan-21-logically-bound-switch:
   summary: Execute logically bound images tests for switching images
@@ -77,13 +79,15 @@ execute:
     how: fmf
     test:
       - /tmt/tests/tests/test-23-install-outside-container
+  extra-works_for_composefs: true
 
 /plan-23-usroverlay:
   summary: Execute tests for bootc usrover
   discover:
     how: fmf
     test:
       - /tmt/tests/tests/test-23-usroverlay
+  extra-works_for_composefs: true
 
 /plan-24-image-upgrade-reboot:
   summary: Execute local upgrade tests
@@ -92,20 +96,23 @@ execute:
     test:
       - /tmt/tests/tests/test-24-image-upgrade-reboot
   extra-try_bind_storage: true
+  extra-works_for_composefs: true
 
 /plan-25-soft-reboot:
   summary: Execute soft reboot test
   discover:
     how: fmf
     test:
       - /tmt/tests/tests/test-25-soft-reboot
+  extra-works_for_composefs: true
 
 /plan-26-download-only-upgrade:
   summary: Execute download-only upgrade tests
   discover:
     how: fmf
     test:
       - /tmt/tests/tests/test-26-download-only-upgrade
+  extra-works_for_composefs: true
 
 /plan-27-custom-selinux-policy:
   summary: Execute custom selinux policy test
@@ -159,6 +166,7 @@ execute:
     how: fmf
     test:
       - /tmt/tests/tests/test-32-install-to-filesystem-var-mount
+  extra-works_for_composefs: true
 
 /plan-33-bib-build:
   summary: Test building a qcow2 disk image with bootc-image-builder

tmt/tests/booted/test-01-readonly.nu

@@ -1,6 +1,7 @@
 # number: 1
 # extra:
 #   try_bind_storage: true
+#   works_for_composefs: true
 # tmt:
 #   summary: Execute booted readonly/nondestructive tests
 #   duration: 30m
@@ -18,4 +19,4 @@ for test_file in $tests {
     nu $test_file
 }
 
-tap ok
+tap ok

tmt/tests/booted/test-download-only-upgrade.nu

@@ -2,6 +2,8 @@
 # tmt:
 #   summary: Execute download-only upgrade tests
 #   duration: 40m
+# extra:
+#   works_for_composefs: true
 #
 # This test does:
 # bootc image copy-to-storage

tmt/tests/booted/test-image-pushpull-upgrade.nu

@@ -2,6 +2,8 @@
 # tmt:
 #   summary: Execute local upgrade tests
 #   duration: 30m
+# extra:
+#   works_for_composefs: true
 #
 # This test does:
 # bootc image copy-to-storage

tmt/tests/booted/test-image-upgrade-reboot.nu

@@ -1,6 +1,7 @@
 # number: 24
 # extra:
 #   try_bind_storage: true
+#   works_for_composefs: true
 # tmt:
 #   summary: Execute local upgrade tests
 #   duration: 30m

tmt/tests/booted/test-install-outside-container.nu

@@ -2,6 +2,8 @@
 # tmt:
 #   summary: Execute tests for installing outside of a container
 #   duration: 30m
+# extra:
+#   works_for_composefs: true
 #
 use std assert
 use tap.nu

tmt/tests/booted/test-install-to-filesystem-var-mount.sh

@@ -1,4 +1,6 @@
 # number: 32
+# extra:
+#   works_for_composefs: true
 # tmt:
 #   summary: Test bootc install to-filesystem with separate /var mount
 #   duration: 30m

tmt/tests/booted/test-soft-reboot.nu

@@ -2,6 +2,8 @@
 # tmt:
 #   summary: Execute soft reboot test
 #   duration: 30m
+# extra:
+#   works_for_composefs: true
 #
 # Verify that soft reboot works (on by default)
 use std assert