feat(resolvables): target config resolution and cascade target deletes

Add support for resolvable target configs ($ref references to resource
properties) and cascade target deletes on destroy.

Target config resolution:
- Targets can reference resource properties via $res/$ref (e.g., a
  Grafana target resolving its endpoint from a Compose stack)
- TargetUpdater FSM gains a Resolving state for config $ref resolution
- DAG edges enforce ordering: target resolves before its resources start
- Resolve cache strips resolvable metadata before passing config to plugins
- Non-blocking retries in resolve cache using SendAfter

Cascade target deletes:
- On destroy, targets with $ref dependencies are automatically deleted
  (plain targets like docker/us-east-1 survive destroy)
- BFS discovery of transitive cascade chains via FindTargetsDependingOnMany
- Cascade-failed targets properly marked as Failed in FormaCommandPersister
- IsCascade/CascadeSource fields on target and resource updates for CLI display

Also includes:
- DB-agnostic idempotent target create (replaces SQLite-specific constraint matching)
- StateNotStarted as initial TargetUpdater FSM state
- Mutation test timeout increased to 90 minutes

Author: JeroenSoeters

.github/workflows/mutation-test-pr.yml

@@ -7,7 +7,7 @@ on:
 jobs:
   mutation-test:
     runs-on: ubuntu-latest
-    timeout-minutes: 60
+    timeout-minutes: 90
     continue-on-error: true
     steps:
       - uses: actions/checkout@v4

internal/cli/destroy/destroy.go

@@ -184,6 +184,25 @@ func runDestroyForHumans(app *app.App, opts *DestroyOptions) error {
 			}
 		}
 
+		hasCascadeTargets := false
+		for _, tu := range res.Simulation.Command.TargetUpdates {
+			if tu.IsCascade {
+				hasCascadeTargets = true
+				break
+			}
+		}
+		if hasCascadeTargets {
+			fmt.Printf("\n%s\n\n", display.Grey("The following targets will be cascade-deleted:"))
+			for _, tu := range res.Simulation.Command.TargetUpdates {
+				if tu.IsCascade {
+					fmt.Printf("  %s %s (depends on %s)\n",
+						display.Red("•"),
+						display.LightBlue(tu.TargetLabel),
+						display.Grey(tu.CascadeSource))
+				}
+			}
+		}
+
 		fmt.Printf("\n%s\n", display.Grey("To proceed with cascade deletes, use --on-dependents=cascade"))
 		return fmt.Errorf("cascade deletes detected, aborting (use --on-dependents=cascade to proceed)")
 	}
@@ -259,12 +278,17 @@ func runDestroyForMachines(app *app.App, opts *DestroyOptions) error {
 	return printer.Print(&apimodel.CommandID{CommandID: res.CommandID})
 }
 
-// hasCascadeDeletes checks if any resource updates in the command are cascade deletes
+// hasCascadeDeletes checks if any resource or target updates in the command are cascade deletes
 func hasCascadeDeletes(cmd *apimodel.Command) bool {
 	for _, ru := range cmd.ResourceUpdates {
 		if ru.IsCascade {
 			return true
 		}
 	}
+	for _, tu := range cmd.TargetUpdates {
+		if tu.IsCascade {
+			return true
+		}
+	}
 	return false
 }

internal/datastore/aurora/aurora.go

@@ -1845,6 +1845,99 @@ func (d *DatastoreAuroraDataAPI) FindResourcesDependingOnMany(ksuids []string) (
 	return result, nil
 }
 
+func (d *DatastoreAuroraDataAPI) FindTargetsDependingOnMany(ksuids []string) (map[string][]*pkgmodel.Target, error) {
+	ctx := context.Background()
+
+	if len(ksuids) == 0 {
+		return make(map[string][]*pkgmodel.Target), nil
+	}
+
+	// Build OR conditions for each KSUID pattern with named parameters.
+	// Use regex to handle PostgreSQL's jsonb::text formatting, which adds spaces after colons.
+	var conditions []string
+	var params []types.SqlParameter
+	for i, ksuid := range ksuids {
+		pattern := fmt.Sprintf(`"\$ref"\s*:\s*"formae://%s#`, ksuid)
+		paramName := fmt.Sprintf("pattern%d", i)
+		conditions = append(conditions, fmt.Sprintf("config::text ~ :%s", paramName))
+		params = append(params, types.SqlParameter{
+			Name:  aws.String(paramName),
+			Value: &types.FieldMemberStringValue{Value: pattern},
+		})
+	}
+
+	query := fmt.Sprintf(`
+	SELECT label, version, namespace, config, discoverable
+	FROM targets t1
+	WHERE (%s)
+	AND NOT EXISTS (
+		SELECT 1
+		FROM targets t2
+		WHERE t1.label = t2.label
+		AND t2.version > t1.version
+	)
+	`, strings.Join(conditions, " OR "))
+
+	output, err := d.executeStatement(ctx, query, params)
+	if err != nil {
+		return nil, err
+	}
+
+	// Build a map of KSUID -> targets that depend on it
+	result := make(map[string][]*pkgmodel.Target)
+	for _, record := range output.Records {
+		if len(record) < 5 {
+			return nil, fmt.Errorf("unexpected record length: %d", len(record))
+		}
+
+		label, err := getStringField(record[0])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse label: %w", err)
+		}
+
+		version, err := getIntField(record[1])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse version: %w", err)
+		}
+
+		namespace, err := getStringField(record[2])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse namespace: %w", err)
+		}
+
+		config, err := getRawJSONField(record[3])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse config: %w", err)
+		}
+
+		discoverable, err := getBoolField(record[4])
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse discoverable: %w", err)
+		}
+
+		target := &pkgmodel.Target{
+			Label:        label,
+			Namespace:    namespace,
+			Config:       config,
+			Discoverable: discoverable,
+			Version:      version,
+		}
+
+		// Find which of the input KSUIDs this target depends on.
+		// jsonb::text output has spaces after colons, so check both forms.
+		configStr := string(config)
+		for _, ksuid := range ksuids {
+			withSpace := fmt.Sprintf("\"$ref\": \"formae://%s#", ksuid)
+			withoutSpace := fmt.Sprintf("\"$ref\":\"formae://%s#", ksuid)
+			if strings.Contains(configStr, withSpace) || strings.Contains(configStr, withoutSpace) {
+				result[ksuid] = append(result[ksuid], target)
+			}
+		}
+	}
+
+	return result, nil
+}
+
 func (d *DatastoreAuroraDataAPI) StoreStack(stack *pkgmodel.Forma, commandID string) (string, error) {
 	var lastVersionID string
 	for _, resource := range stack.Resources {

internal/datastore/datastore.go

@@ -153,6 +153,9 @@ type Datastore interface {
 	// FindResourcesDependingOnMany returns all resources that reference any of the given resources via $ref.
 	// Returns a map from referenced KSUID to the resources that depend on it.
 	FindResourcesDependingOnMany(ksuids []string) (map[string][]*pkgmodel.Resource, error)
+	// FindTargetsDependingOnMany returns all targets whose config references any of the given resources via $ref.
+	// Returns a map from source KSUID to the list of dependent targets.
+	FindTargetsDependingOnMany(ksuids []string) (map[string][]*pkgmodel.Target, error)
 
 	// Resource-by-stack operations - query resources grouped by stack
 

internal/datastore/mock_datastore_test.go

@@ -71,6 +71,9 @@ func (m *mockDatastore) FindResourcesDependingOn(_ string) ([]*pkgmodel.Resource
 func (m *mockDatastore) FindResourcesDependingOnMany(_ []string) (map[string][]*pkgmodel.Resource, error) {
 	return nil, nil
 }
+func (m *mockDatastore) FindTargetsDependingOnMany(_ []string) (map[string][]*pkgmodel.Target, error) {
+	return make(map[string][]*pkgmodel.Target), nil
+}
 func (m *mockDatastore) BulkStoreResources(_ []pkgmodel.Resource, _ string) (string, error) {
 	return "", nil
 }

internal/datastore/postgres/postgres.go

@@ -1174,6 +1174,76 @@ func (d DatastorePostgres) FindResourcesDependingOnMany(ksuids []string) (map[st
 	return result, nil
 }
 
+func (d DatastorePostgres) FindTargetsDependingOnMany(ksuids []string) (map[string][]*pkgmodel.Target, error) {
+	ctx, span := tracer.Start(context.Background(), "FindTargetsDependingOnMany")
+	defer span.End()
+
+	if len(ksuids) == 0 {
+		return make(map[string][]*pkgmodel.Target), nil
+	}
+
+	// Build OR conditions for each KSUID pattern with numbered placeholders
+	// Use regex to handle Postgres JSONB text formatting which adds spaces after colons
+	var conditions []string
+	var args []any
+	for i, ksuid := range ksuids {
+		pattern := fmt.Sprintf(`"\$ref"\s*:\s*"formae://%s#`, ksuid)
+		conditions = append(conditions, fmt.Sprintf("config::text ~ $%d", i+1))
+		args = append(args, pattern)
+	}
+
+	query := fmt.Sprintf(`
+	SELECT label, version, namespace, config, discoverable
+	FROM targets t1
+	WHERE (%s)
+	AND NOT EXISTS (
+		SELECT 1
+		FROM targets t2
+		WHERE t1.label = t2.label
+		AND t2.version > t1.version
+	)
+	`, strings.Join(conditions, " OR "))
+
+	rows, err := d.pool.Query(ctx, query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	// Build a map of KSUID -> targets that depend on it
+	result := make(map[string][]*pkgmodel.Target)
+	for rows.Next() {
+		var label, namespace string
+		var version int
+		var config json.RawMessage
+		var discoverable bool
+		if err := rows.Scan(&label, &version, &namespace, &config, &discoverable); err != nil {
+			return nil, err
+		}
+
+		target := &pkgmodel.Target{
+			Label:        label,
+			Namespace:    namespace,
+			Config:       config,
+			Discoverable: discoverable,
+			Version:      version,
+		}
+
+		// Find which of the input KSUIDs this target depends on
+		// jsonb::text output has spaces after colons, so check both forms.
+		configStr := string(config)
+		for _, ksuid := range ksuids {
+			withSpace := fmt.Sprintf("\"$ref\": \"formae://%s#", ksuid)
+			withoutSpace := fmt.Sprintf("\"$ref\":\"formae://%s#", ksuid)
+			if strings.Contains(configStr, withSpace) || strings.Contains(configStr, withoutSpace) {
+				result[ksuid] = append(result[ksuid], target)
+			}
+		}
+	}
+
+	return result, nil
+}
+
 func (d DatastorePostgres) LoadResourceByNativeID(nativeID string, resourceType string) (*pkgmodel.Resource, error) {
 	ctx, span := tracer.Start(context.Background(), "LoadResourceByNativeID")
 	defer span.End()

internal/datastore/sqlite/sqlite.go

@@ -3117,6 +3117,79 @@ func (d DatastoreSQLite) FindResourcesDependingOnMany(ksuids []string) (map[stri
 	return result, nil
 }
 
+func (d DatastoreSQLite) FindTargetsDependingOnMany(ksuids []string) (map[string][]*pkgmodel.Target, error) {
+	slog.Debug("SQLite START", "method", "FindTargetsDependingOnMany", "ksuids", len(ksuids))
+	start := time.Now()
+	defer func() {
+		slog.Debug("SQLite END", "method", "FindTargetsDependingOnMany", "ksuids", len(ksuids), "duration", time.Since(start))
+	}()
+	_, span := sqliteTracer.Start(context.Background(), "FindTargetsDependingOnMany")
+	defer span.End()
+
+	if len(ksuids) == 0 {
+		return make(map[string][]*pkgmodel.Target), nil
+	}
+
+	// Build OR conditions for each KSUID pattern
+	// The format is: "$ref":"formae://KSUID#/..."
+	var conditions []string
+	var args []any
+	for _, ksuid := range ksuids {
+		pattern := fmt.Sprintf("%%\"$ref\":\"formae://%s#%%", ksuid)
+		conditions = append(conditions, "config LIKE ?")
+		args = append(args, pattern)
+	}
+
+	query := fmt.Sprintf(`
+	SELECT label, version, namespace, config, discoverable
+	FROM targets t1
+	WHERE (%s)
+	AND NOT EXISTS (
+		SELECT 1
+		FROM targets t2
+		WHERE t1.label = t2.label
+		AND t2.version > t1.version
+	)
+	`, strings.Join(conditions, " OR "))
+
+	rows, err := d.conn.Query(query, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer closeRows(rows)
+
+	// Build a map of KSUID -> targets that depend on it
+	result := make(map[string][]*pkgmodel.Target)
+	for rows.Next() {
+		var label, namespace string
+		var version int
+		var config json.RawMessage
+		var discoverable int
+		if err := rows.Scan(&label, &version, &namespace, &config, &discoverable); err != nil {
+			return nil, err
+		}
+
+		target := &pkgmodel.Target{
+			Label:        label,
+			Namespace:    namespace,
+			Config:       config,
+			Discoverable: discoverable == 1,
+			Version:      version,
+		}
+
+		// Find which of the input KSUIDs this target depends on
+		configStr := string(config)
+		for _, ksuid := range ksuids {
+			pattern := fmt.Sprintf("\"$ref\":\"formae://%s#", ksuid)
+			if strings.Contains(configStr, pattern) {
+				result[ksuid] = append(result[ksuid], target)
+			}
+		}
+	}
+
+	return result, nil
+}
+
 func (d DatastoreSQLite) GetKSUIDByTriplet(stack, label, resourceType string) (string, error) {
 	_, span := sqliteTracer.Start(context.Background(), "GetKSUIDByTriplet")
 	defer span.End()