[WIP] Webview

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "examples/bmeg-dictionary"]
-	path = examples/bmeg-dictionary
-	url = https://github.com/bmeg/bmeg-dictionary.git

README.md

@@ -18,10 +18,9 @@ name: census_2010
 
 params:
   census: 
-    type: File
+    type: file
     default: ../data/census_2010_byzip.json
   date: 
-    type: string
     default: "2010-01-01"
   schema: 
     type: path
@@ -37,6 +36,9 @@ outputs:
     json:
       path: census_data.ndjson
 
+outputs:
+
+
 pipelines:
   transform:
     - from: censusData

cmd/root.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/bmeg/sifter/cmd/inspect"
 	"github.com/bmeg/sifter/cmd/run"
+	"github.com/bmeg/sifter/cmd/web"
 	"github.com/spf13/cobra"
 )
 
@@ -18,6 +19,7 @@ var RootCmd = &cobra.Command{
 func init() {
 	RootCmd.AddCommand(run.Cmd)
 	RootCmd.AddCommand(inspect.Cmd)
+	RootCmd.AddCommand(web.Cmd)
 }
 
 var genBashCompletionCmd = &cobra.Command{

cmd/run/main.go

@@ -14,6 +14,8 @@ var outDir string = ""
 var paramsFile string = ""
 var verbose bool = false
 var cmdParams map[string]string
+var captureDir string = ""
+var captureLimit int = 10
 
 // Cmd is the declaration of the command line
 var Cmd = &cobra.Command{
@@ -46,11 +48,11 @@ var Cmd = &cobra.Command{
 				}
 				pb := playbook.Playbook{}
 				playbook.ParseBytes(yaml, "./playbook.yaml", &pb)
-				if err := Execute(pb, "./", "./", outDir, params); err != nil {
+				if err := Execute(pb, "./", "./", outDir, params, captureDir, captureLimit); err != nil {
 					return err
 				}
 			} else {
-				if err := ExecuteFile(playFile, "./", outDir, params); err != nil {
+				if err := ExecuteFile(playFile, "./", outDir, params, captureDir, captureLimit); err != nil {
 					return err
 				}
 			}
@@ -65,4 +67,7 @@ func init() {
 	flags.BoolVarP(&verbose, "verbose", "v", verbose, "Verbose logging")
 	flags.StringToStringVarP(&cmdParams, "param", "p", cmdParams, "Parameter variable")
 	flags.StringVarP(&paramsFile, "params-file", "f", paramsFile, "Parameter file")
+	flags.StringVarP(&captureDir, "capture-dir", "d", "", "Directory for capture files (default: None)")
+	flags.IntVarP(&captureLimit, "capture-limit", "l", 10, "Max records to capture per step (0 = unlimited)")
+	flags.StringVarP(&outDir, "output", "o", outDir, "Output directory for playbook results (default: current directory or value specified in playbook)")
 }

cmd/run/run.go

@@ -1,6 +1,7 @@
 package run
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 
@@ -9,7 +10,7 @@ import (
 	"github.com/bmeg/sifter/task"
 )
 
-func ExecuteFile(playFile string, workDir string, outDir string, inputs map[string]string) error {
+func ExecuteFile(playFile string, workDir string, outDir string, inputs map[string]string, debugDir string, debugLimit int) error {
 	logger.Info("Starting", "playFile", playFile)
 	pb := playbook.Playbook{}
 	if err := playbook.ParseFile(playFile, &pb); err != nil {
@@ -19,10 +20,10 @@ func ExecuteFile(playFile string, workDir string, outDir string, inputs map[stri
 	a, _ := filepath.Abs(playFile)
 	baseDir := filepath.Dir(a)
 	logger.Debug("parsed file", "baseDir", baseDir, "playbook", pb)
-	return Execute(pb, baseDir, workDir, outDir, inputs)
+	return Execute(pb, baseDir, workDir, outDir, inputs, debugDir, debugLimit)
 }
 
-func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string, params map[string]string) error {
+func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string, params map[string]string, debugDir string, debugLimit int) error {
 
 	if outDir == "" {
 		outDir = pb.GetDefaultOutDir()
@@ -32,13 +33,39 @@ func Execute(pb playbook.Playbook, baseDir string, workDir string, outDir string
 		os.MkdirAll(outDir, 0777)
 	}
 
+	// Setup debug capture directory if enabled
+	// Enable if: user explicitly set dir, OR user changed limit from default
+	enableDebug := debugDir != "" || (debugLimit != 10)
+	if enableDebug {
+		if debugDir == "" {
+			debugDir = filepath.Join(workDir, "debug-capture")
+		} else if !filepath.IsAbs(debugDir) {
+			debugDir = filepath.Join(workDir, debugDir)
+		}
+		if info, err := os.Stat(debugDir); err != nil {
+			if os.IsNotExist(err) {
+				if mkErr := os.MkdirAll(debugDir, 0777); mkErr != nil {
+					logger.Error("Failed to create debug directory", "error", mkErr)
+					return mkErr
+				}
+			} else {
+				logger.Error("Failed to access debug directory", "path", debugDir, "error", err)
+				return err
+			}
+		} else if !info.IsDir() {
+			logger.Error("Debug path exists but is not a directory", "path", debugDir)
+			return fmt.Errorf("debug path %s exists but is not a directory", debugDir)
+		}
+		logger.Info("Debug capture enabled", "dir", debugDir, "limit", debugLimit)
+	}
+
 	nInputs, err := pb.PrepConfig(params, workDir)
 	if err != nil {
 		return err
 	}
 	logger.Debug("Running", "outDir", outDir)
 
 	t := task.NewTask(pb.Name, baseDir, workDir, outDir, nInputs)
-	err = pb.Execute(t)
+	err = pb.ExecuteWithCapture(t, debugDir, debugLimit)
 	return err
 }

cmd/web/main.go

@@ -0,0 +1,183 @@
+package web
+
+import (
+	"embed"
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"sigs.k8s.io/yaml"
+
+	"github.com/spf13/cobra"
+)
+
+//go:embed static/*
+var staticFS embed.FS
+
+var playbookDir string
+var siteDir string
+var port string = "8081"
+
+// Cmd is the declaration of the command line
+var Cmd = &cobra.Command{
+	Use:   "web <script>",
+	Short: "View sifter script in browser",
+	Args:  cobra.MinimumNArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		playbookDir = args[0]
+
+		var httpFS http.FileSystem
+
+		if siteDir == "" {
+			// Serve embedded static files
+			staticFiles, err := fs.Sub(staticFS, "static")
+			if err != nil {
+				log.Fatalf("failed to create sub FS: %v", err)
+			}
+			httpFS = http.FS(staticFiles)
+		} else {
+			httpFS = http.Dir(siteDir)
+		}
+		http.Handle("/", http.FileServer(httpFS))
+
+		// API endpoints
+		ph := playbookHandler{playbookDir}
+		http.HandleFunc("/api/files", ph.listFilesHandler)
+		http.HandleFunc("/api/playbook", ph.getPlaybookHandler)
+
+		fmt.Printf("Server listening on http://localhost:%s\n", port)
+		log.Fatal(http.ListenAndServe(":"+port, nil))
+
+		return nil
+	},
+}
+
+type playbookHandler struct {
+	baseDir string
+}
+
+type fileTreeNode struct {
+	Name     string         `json:"name"`
+	Path     string         `json:"path"`
+	IsDir    bool           `json:"isDir"`
+	Children []fileTreeNode `json:"children,omitempty"`
+}
+
+func buildFileTree(absDir string, relativeDir string) ([]fileTreeNode, error) {
+	entries, err := os.ReadDir(absDir)
+	if err != nil {
+		return nil, err
+	}
+
+	sort.Slice(entries, func(i, j int) bool {
+		if entries[i].IsDir() != entries[j].IsDir() {
+			return entries[i].IsDir()
+		}
+		return entries[i].Name() < entries[j].Name()
+	})
+
+	nodes := make([]fileTreeNode, 0, len(entries))
+	for _, entry := range entries {
+		entryRelPath := filepath.Join(relativeDir, entry.Name())
+		node := fileTreeNode{
+			Name:  entry.Name(),
+			Path:  filepath.ToSlash(entryRelPath),
+			IsDir: entry.IsDir(),
+		}
+
+		if entry.IsDir() {
+			children, err := buildFileTree(filepath.Join(absDir, entry.Name()), entryRelPath)
+			if err != nil {
+				return nil, err
+			}
+			node.Children = children
+		}
+
+		nodes = append(nodes, node)
+	}
+
+	return nodes, nil
+}
+
+func (ph *playbookHandler) listFilesHandler(w http.ResponseWriter, r *http.Request) {
+	basePath, err := filepath.Abs(ph.baseDir)
+	if err != nil {
+		http.Error(w, "failed to resolve playbook directory", http.StatusInternalServerError)
+		return
+	}
+
+	entries, err := buildFileTree(basePath, "")
+
+	if err != nil {
+		http.Error(w, "failed to read playbook directory", http.StatusInternalServerError)
+		return
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(entries)
+}
+
+func (ph *playbookHandler) getPlaybookHandler(w http.ResponseWriter, r *http.Request) {
+	name := r.URL.Query().Get("name")
+	if name == "" {
+		http.Error(w, "missing name parameter", http.StatusBadRequest)
+		return
+	}
+
+	// Allow nested relative paths while preventing traversal and absolute paths.
+	cleanName := filepath.Clean(name)
+	if cleanName == "." || strings.HasPrefix(cleanName, "..") || filepath.IsAbs(cleanName) {
+		http.Error(w, "invalid playbook name", http.StatusBadRequest)
+		return
+	}
+
+	baseAbsPath, err := filepath.Abs(ph.baseDir)
+	if err != nil {
+		http.Error(w, "failed to resolve playbook directory", http.StatusInternalServerError)
+		return
+	}
+
+	path := filepath.Join(ph.baseDir, cleanName)
+	targetAbsPath, err := filepath.Abs(path)
+	if err != nil {
+		http.Error(w, "invalid playbook path", http.StatusBadRequest)
+		return
+	}
+
+	basePrefix := baseAbsPath + string(os.PathSeparator)
+	if targetAbsPath != baseAbsPath && !strings.HasPrefix(targetAbsPath, basePrefix) {
+		http.Error(w, "invalid playbook name", http.StatusBadRequest)
+		return
+	}
+
+	content, err := os.ReadFile(path)
+	if err != nil {
+		http.Error(w, "playbook not found", http.StatusNotFound)
+		return
+	}
+	format := r.URL.Query().Get("format")
+	if format == "json" {
+		jsonBytes, err := yaml.YAMLToJSON(content)
+		if err != nil {
+			http.Error(w, "failed to convert yaml to json", http.StatusInternalServerError)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		w.Write(jsonBytes)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	w.Write(content)
+}
+
+func init() {
+	flags := Cmd.Flags()
+	flags.StringVarP(&siteDir, "site-dir", "s", siteDir, "Serve Custom site dir")
+	flags.StringVarP(&port, "port", "p", port, "Port to listen on")
+}

cmd/web/static/app.js

@@ -0,0 +1,33 @@
+document.addEventListener('DOMContentLoaded', () => {
+    const listEl = document.getElementById('playbook-list');
+    const contentEl = document.getElementById('playbook-content');
+
+    // Fetch list of playbooks
+    fetch('/api/playbooks')
+        .then(res => res.json())
+        .then(playbooks => {
+            playbooks.forEach(name => {
+                const li = document.createElement('li');
+                li.textContent = name;
+                li.addEventListener('click', () => loadPlaybook(name, li));
+                listEl.appendChild(li);
+            });
+        })
+        .catch(err => console.error('Failed to load playbook list', err));
+
+    function loadPlaybook(name, element) {
+        // Highlight selected
+        Array.from(listEl.children).forEach(child => child.classList.remove('selected'));
+        element.classList.add('selected');
+        fetch(`/api/playbook?name=${encodeURIComponent(name)}`)
+            .then(res => {
+                if (!res.ok) throw new Error('Playbook not found');
+                return res.text();
+            })
+            .then(text => {
+                contentEl.textContent = text;
+                hljs.highlightElement(contentEl);
+            })
+            .catch(err => console.error('Failed to load playbook', err));
+    }
+});

cmd/web/static/index.html

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Sifter Playbook Viewer</title>
+    <link rel="stylesheet" href="style.css" />
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;600&display=swap" rel="stylesheet" />
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>
+    <link rel="stylesheet"
+        href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/styles/github-dark.min.css" />
+</head>
+
+<body>
+    <div class="container">
+        <aside class="sidebar">
+            <h1>Sifter Playbooks</h1>
+            <ul id="playbook-list"></ul>
+        </aside>
+        <main class="content">
+            <pre><code id="playbook-content" class="yaml"></code></pre>
+        </main>
+    </div>
+    <script src="app.js"></script>
+    <script>hljs.highlightAll();</script>
+</body>
+
+</html>