Support new file/format model shape for local_server.kv_store and secret_store (as well as metadata in kv_store) (#1446)

* Support new file/format model shape for local_server.kv_store and secret_store 
* Support new metadata field in model shape for local_server.kv_store

Author: harmony7

CHANGELOG.md

@@ -6,6 +6,9 @@
 
 ### Enhancements:
 
+- feat(config): Support file/format for kv_store and secret_store in fastly.toml
+- feat(config): Support metadata for kv_store in fastly.toml
+
 ### Bug fixes:
 
 ### Dependencies:

pkg/manifest/file.go

@@ -2,6 +2,7 @@ package manifest
 
 import (
 	"bufio"
+	"bytes"
 	"fmt"
 	"io"
 	"os"
@@ -50,6 +51,108 @@ type File struct {
 	readError error
 }
 
+// MarshalTOML performs custom marshalling to TOML for objects of File type.
+func (f *File) MarshalTOML() ([]byte, error) {
+	localServer := make(map[string]any)
+
+	if f.LocalServer.Backends != nil {
+		localServer["backends"] = f.LocalServer.Backends
+	}
+
+	if f.LocalServer.ConfigStores != nil {
+		localServer["config_stores"] = f.LocalServer.ConfigStores
+	}
+
+	if f.LocalServer.KVStores != nil {
+		kvStores := make(map[string]any)
+		for key, entry := range f.LocalServer.KVStores {
+			if entry.External != nil {
+				kvStores[key] = map[string]any{
+					"file":   entry.External.File,
+					"format": entry.External.Format,
+				}
+			} else {
+				items := make([]map[string]any, 0, len(entry.Array))
+				for _, e := range entry.Array {
+					obj := map[string]any{"key": e.Key}
+					if e.File != "" {
+						obj["file"] = e.File
+					}
+					if e.Data != "" {
+						obj["data"] = e.Data
+					}
+					if e.Metadata != "" {
+						obj["metadata"] = e.Metadata
+					}
+					items = append(items, obj)
+				}
+				kvStores[key] = items
+			}
+		}
+		localServer["kv_stores"] = kvStores
+	}
+
+	if f.LocalServer.SecretStores != nil {
+		secretStores := make(map[string]any)
+		for key, entry := range f.LocalServer.SecretStores {
+			if entry.External != nil {
+				secretStores[key] = map[string]any{
+					"file":   entry.External.File,
+					"format": entry.External.Format,
+				}
+			} else {
+				items := make([]map[string]any, 0, len(entry.Array))
+				for _, e := range entry.Array {
+					obj := map[string]any{"key": e.Key}
+					if e.File != "" {
+						obj["file"] = e.File
+					}
+					if e.Data != "" {
+						obj["data"] = e.Data
+					}
+					items = append(items, obj)
+				}
+				secretStores[key] = items
+			}
+		}
+		localServer["secret_stores"] = secretStores
+	}
+
+	if f.LocalServer.ViceroyVersion != "" {
+		localServer["viceroy_version"] = f.LocalServer.ViceroyVersion
+	}
+
+	out := struct {
+		Authors         []string `toml:"authors"`
+		ClonedFrom      string   `toml:"cloned_from,omitempty"`
+		Description     string   `toml:"description"`
+		Language        string   `toml:"language"`
+		Profile         string   `toml:"profile,omitempty"`
+		LocalServer     any      `toml:"local_server"` // override this field
+		ManifestVersion Version  `toml:"manifest_version"`
+		Name            string   `toml:"name"`
+		Scripts         Scripts  `toml:"scripts,omitempty"`
+		ServiceID       string   `toml:"service_id"`
+		Setup           Setup    `toml:"setup,omitempty"`
+	}{
+		Authors:         f.Authors,
+		ClonedFrom:      f.ClonedFrom,
+		Description:     f.Description,
+		Language:        f.Language,
+		Profile:         f.Profile,
+		LocalServer:     localServer,
+		ManifestVersion: f.ManifestVersion,
+		Name:            f.Name,
+		Scripts:         f.Scripts,
+		ServiceID:       f.ServiceID,
+		Setup:           f.Setup,
+	}
+
+	var buf bytes.Buffer
+	err := toml.NewEncoder(&buf).Encode(out)
+	return buf.Bytes(), err
+}
+
 // Exists yields whether the manifest exists.
 //
 // Specifically, it indicates that a toml.Unmarshal() of the toml disk content

pkg/manifest/local_server.go

@@ -1,12 +1,19 @@
 package manifest
 
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/pelletier/go-toml"
+)
+
 // LocalServer represents a list of mocked Viceroy resources.
 type LocalServer struct {
-	Backends       map[string]LocalBackend       `toml:"backends"`
-	ConfigStores   map[string]LocalConfigStore   `toml:"config_stores,omitempty"`
-	KVStores       map[string][]LocalKVStore     `toml:"kv_stores,omitempty"`
-	SecretStores   map[string][]LocalSecretStore `toml:"secret_stores,omitempty"`
-	ViceroyVersion string                        `toml:"viceroy_version,omitempty"`
+	Backends       map[string]LocalBackend     `toml:"backends"`
+	ConfigStores   map[string]LocalConfigStore `toml:"config_stores,omitempty"`
+	KVStores       LocalKVStoreMap             `toml:"kv_stores,omitempty"`
+	SecretStores   LocalSecretStoreMap         `toml:"secret_stores,omitempty"`
+	ViceroyVersion string                      `toml:"viceroy_version,omitempty"`
 }
 
 // LocalBackend represents a backend to be mocked by the local testing server.
@@ -24,16 +31,172 @@ type LocalConfigStore struct {
 	Contents map[string]string `toml:"contents,omitempty"`
 }
 
-// LocalKVStore represents an kv_store to be mocked by the local testing server.
+// KVStoreArrayEntry represents an array-based key/value store entries.
+// It expects a key plus either a data or file field.
+type KVStoreArrayEntry struct {
+	Key      string `toml:"key"`
+	File     string `toml:"file,omitempty"`
+	Data     string `toml:"data,omitempty"`
+	Metadata string `toml:"metadata,omitempty"`
+}
+
+// KVStoreExternalFile represents the external key/value store,
+// which must have both a file and a format.
+type KVStoreExternalFile struct {
+	File   string `toml:"file"`
+	Format string `toml:"format"`
+}
+
+// LocalKVStore represents a kv_store to be mocked by the local testing server.
+// It is a union type and can either be an array of KVStoreArrayEntry or a single KVStoreExternalFile.
+// The IsArray flag is used to preserve the original input style.
 type LocalKVStore struct {
+	IsArray  bool                 `toml:"-"`
+	Array    []KVStoreArrayEntry  `toml:"-"`
+	External *KVStoreExternalFile `toml:"-"`
+}
+
+// LocalKVStoreMap is a map of kv_store names to the local kv_store representation.
+type LocalKVStoreMap map[string]LocalKVStore
+
+// UnmarshalTOML performs custom unmarshalling of TOML data for LocalKVStoreMap.
+func (m *LocalKVStoreMap) UnmarshalTOML(v any) error {
+	raw, ok := v.(map[string]any)
+	if !ok {
+		return fmt.Errorf("expected kv_stores to be a TOML table")
+	}
+
+	result := make(LocalKVStoreMap)
+
+	for key, val := range raw {
+		switch typed := val.(type) {
+		case []any:
+			var entries []KVStoreArrayEntry
+			for _, item := range typed {
+				obj, ok := item.(map[string]any)
+				if !ok {
+					return fmt.Errorf("invalid item in array for key %q", key)
+				}
+				var arrayEntry KVStoreArrayEntry
+				if err := decodeTOMLMap(obj, &arrayEntry); err != nil {
+					return fmt.Errorf("decode failed for array item in key %q: %w", key, err)
+				}
+				entries = append(entries, arrayEntry)
+			}
+			result[key] = LocalKVStore{
+				IsArray: true,
+				Array:   entries,
+			}
+
+		case map[string]any:
+			file, hasFile := typed["file"].(string)
+			format, hasFormat := typed["format"].(string)
+
+			if !hasFile || !hasFormat {
+				return fmt.Errorf("key %q must have both file and format", key)
+			}
+			result[key] = LocalKVStore{
+				IsArray: false,
+				External: &KVStoreExternalFile{
+					File:   file,
+					Format: format,
+				},
+			}
+
+		default:
+			return fmt.Errorf("unsupported value type for key %q: %T", key, typed)
+		}
+	}
+
+	*m = result
+	return nil
+}
+
+// SecretStoreArrayEntry represents an array-based key/value store entries.
+// It expects a key plus either a data or file field.
+type SecretStoreArrayEntry struct {
 	Key  string `toml:"key"`
 	File string `toml:"file,omitempty"`
 	Data string `toml:"data,omitempty"`
 }
 
+// SecretStoreExternalFile represents the external key/value store,
+// which must have both a file and a format.
+type SecretStoreExternalFile struct {
+	File   string `toml:"file"`
+	Format string `toml:"format"`
+}
+
 // LocalSecretStore represents a secret_store to be mocked by the local testing server.
+// It is a union type and can either be an array of SecretStoreArrayEntry or a single SecretStoreExternalFile.
+// The IsArray flag is used to preserve the original input style.
 type LocalSecretStore struct {
-	Key  string `toml:"key"`
-	File string `toml:"file,omitempty"`
-	Data string `toml:"data,omitempty"`
+	IsArray  bool                     `toml:"-"`
+	Array    []SecretStoreArrayEntry  `toml:"-"`
+	External *SecretStoreExternalFile `toml:"-"`
+}
+
+// LocalSecretStoreMap is a map of secret_store names to the local secret_store representation.
+type LocalSecretStoreMap map[string]LocalSecretStore
+
+// UnmarshalTOML performs custom unmarshalling of TOML data for LocalSecretStoreMap.
+func (m *LocalSecretStoreMap) UnmarshalTOML(v any) error {
+	raw, ok := v.(map[string]any)
+	if !ok {
+		return fmt.Errorf("expected secret_stores to be a TOML table")
+	}
+
+	result := make(LocalSecretStoreMap)
+
+	for key, val := range raw {
+		switch typed := val.(type) {
+		case []any:
+			var entries []SecretStoreArrayEntry
+			for _, item := range typed {
+				obj, ok := item.(map[string]any)
+				if !ok {
+					return fmt.Errorf("invalid item in array for key %q", key)
+				}
+				var arrayEntry SecretStoreArrayEntry
+				if err := decodeTOMLMap(obj, &arrayEntry); err != nil {
+					return fmt.Errorf("decode failed for array item in key %q: %w", key, err)
+				}
+				entries = append(entries, arrayEntry)
+			}
+			result[key] = LocalSecretStore{
+				IsArray: true,
+				Array:   entries,
+			}
+
+		case map[string]any:
+			file, hasFile := typed["file"].(string)
+			format, hasFormat := typed["format"].(string)
+
+			if !hasFile || !hasFormat {
+				return fmt.Errorf("key %q must have both file and format", key)
+			}
+			result[key] = LocalSecretStore{
+				IsArray: false,
+				External: &SecretStoreExternalFile{
+					File:   file,
+					Format: format,
+				},
+			}
+
+		default:
+			return fmt.Errorf("unsupported value type for key %q: %T", key, typed)
+		}
+	}
+
+	*m = result
+	return nil
+}
+
+func decodeTOMLMap(m map[string]any, out any) error {
+	buf := new(bytes.Buffer)
+	enc := toml.NewEncoder(buf)
+	if err := enc.Encode(m); err != nil {
+		return err
+	}
+	return toml.NewDecoder(buf).Decode(out)
 }