feat: new queue manager (#36)

* chore: refactor queue manager fully

* chore: better queue manager

* chore: init to idle

Author: MarcStdt

.docker/entrypoint.sh

@@ -71,9 +71,9 @@ if [ -z "$input" ] || [[ $input =~ ([^/]+)/([^:]+):([^/]+) ]]; then
         args+=("--port" "${DRUID_PORT}")
     fi
 
-    if [ ! -z "${DRUID_INIT}" ];
+    if [ ! -z "${DRUID_IDLE}" ];
     then
-        args+=("--init")
+        args+=("--idle")
     fi
 
     echo "Running druid with args from env: ${args[@]}"

.gitignore

@@ -9,6 +9,7 @@ dlv.log
 .DS_Store
 
 __debug_bin*
+debug**
 
 druid-cli-test
 

.vscode/launch.json

@@ -161,6 +161,18 @@
             "env": {},
             "args": [],
             "showLog": true
-        }       
+        },
+		{
+			"name": "Attach",
+			"type": "go",
+			"request": "attach",
+			"mode": "remote",
+			"remotePath": "",
+			"port":2345,
+			"host":"127.0.0.1",
+			"showLog": true,
+			"trace": "log",
+			"logOutput": "rpc"
+		}
     ]
 }

Dockerfile.testing

@@ -14,15 +14,14 @@ RUN apt-get update && \
     apt-get install -y ant && \
     apt-get clean;
 
-
 #/root/go/bin is not in the path
 ENV PATH=$PATH:/root/go/bin
 ENV PATH=$PATH:/usr/local/go/bin
 
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
+
 # Copy go mod and sum files 
 COPY go.mod go.sum ./
 
 # Download all dependencies. Dependencies will be cached if the go.mod and go.sum files are not changed 
-RUN go mod download 
-
-COPY . . 
+RUN go mod download 

Makefile

@@ -39,4 +39,9 @@ test-integration:
 
 test-integration-docker:
 	docker build . -f Dockerfile.testing -t druid-cli-test
-	docker run --rm druid-cli-test bash -c "go test -v ./test/integration"
+	docker run -v ./:/app --entrypoint=/bin/bash --rm druid-cli-test -c "go test -v ./test/integration"
+	docker run -v ./:/app --entrypoint=/bin/bash --rm druid-cli-test -c "go test -v ./test/integration/commands"
+
+test-integration-docker-debug:
+	docker build . -f Dockerfile.testing -t druid-cli-test
+	docker run -v ./:/app --entrypoint=/bin/bash --rm -p 2345:2345 -it druid-cli-test -c "dlv --listen=:2345 --headless=true --log=true --log-output=debugger,debuglineerr,gdbwire,lldbout,rpc --accept-multiclient --api-version=2 test ./test/integration/commands"

cmd/run.go

@@ -58,7 +58,7 @@ var RunCmd = &cobra.Command{
 
 		command := args[0]
 
-		err = queueManager.AddItem(command, false)
+		err = queueManager.AddTempItem(command)
 		return err
 	},
 }

cmd/serve.go

@@ -22,7 +22,7 @@ var ignoreVersionCheck bool
 var port int
 var shutdownWait int
 var additionalEndpoints []string
-var initScroll bool
+var idleScroll bool
 
 var ServeCommand = &cobra.Command{
 	Use:   "serve",
@@ -52,6 +52,12 @@ to interact and monitor the Scroll Application`,
 		logManager := services.NewLogManager()
 		consoleService := services.NewConsoleManager(logManager)
 		processMonitor := services.NewProcessMonitor()
+		ctx := cmd.Context()
+		go func() {
+			<-ctx.Done()
+			processMonitor.ShutdownPromMetrics()
+		}()
+
 		processManager := services.NewProcessManager(logManager, consoleService, processMonitor)
 
 		pluginManager := services.NewPluginManager()
@@ -110,35 +116,26 @@ to interact and monitor the Scroll Application`,
 
 		signals.SetupSignals(queueManager, processManager, a, shutdownWait)
 
-		currentScroll, lock, err := scrollService.Bootstrap(ignoreVersionCheck)
-		if err != nil {
-			return err
-		}
+		if !idleScroll {
 
-		//normal or first launch?
-		if len(lock.Statuses) > 0 {
-			logger.Log().Info("Found lock file, bootstrapping done")
-			err = scrollService.RenderCwdTemplates()
+			currentScroll, lock, err := scrollService.Bootstrap(ignoreVersionCheck)
 			if err != nil {
 				return err
 			}
-			//important to launch plugins, after the templates are rendered, sothat templates can provide for plugins
-			err = processLauncher.LaunchPlugins()
 
-			if err != nil {
-				return err
-			}
+			newScroll := len(lock.Statuses) == 0
 
-			logger.Log().Info("Starting queue manager")
-			go queueManager.Work()
-		} else if initScroll || true { //TODO: remove true
-			logger.Log().Info("No lock file found, but init command available. Bootstrapping...")
+			if newScroll {
+				logger.Log().Info("No lock file found, but init command available. Bootstrapping...")
 
-			logger.Log().Info("Creating lock and bootstrapping files")
-			//There is an error here. We need to bootstrap the files before we render out the templates in the bootstrap func above
-			err := scrollService.CreateLockAndBootstrapFiles()
-			if err != nil {
-				return err
+				logger.Log().Info("Creating lock and bootstrapping files")
+				//There is an error here. We need to bootstrap the files before we render out the templates in the bootstrap func above
+				err := scrollService.CreateLockAndBootstrapFiles()
+				if err != nil {
+					return err
+				}
+			} else {
+				logger.Log().Info("Found lock file, bootstrapping done")
 			}
 
 			logger.Log().Info("Rendering cwd templates")
@@ -158,43 +155,44 @@ to interact and monitor the Scroll Application`,
 			logger.Log().Info("Starting queue manager")
 			go queueManager.Work()
 
-			logger.Log().Info("Starting scroll.init process")
-			//start scroll.init process
-			//initialize if nothing is there
-			err = queueManager.AddItem(currentScroll.Init, true)
+			if newScroll {
+				logger.Log().Info("Starting scroll.init process")
+				//start scroll.init process
+				//initialize if nothing is there
+				err = queueManager.AddAndRememberItem(currentScroll.Init)
+				if err != nil {
+					return err
+				}
+
+				logger.Log().Info("Writing new scroll lock")
+				scrollService.WriteNewScrollLock()
+
+				logger.Log().Info("Bootstrapping done")
+			}
+
+			err = queueManager.QueueLockFile()
 			if err != nil {
 				return err
 			}
 
-			logger.Log().Info("Writing new scroll lock")
-			scrollService.WriteNewScrollLock()
+			//schedule crons
+			logger.Log().Info("Schedule crons")
 
-			logger.Log().Info("Bootstrapping done")
-		}
+			cronManager := services.NewCronManager(currentScroll.Cronjobs, queueManager)
+			err = cronManager.Init()
 
-		err = queueManager.QueueLockFile()
-		if err != nil {
-			return err
-		}
-
-		//schedule crons
-		logger.Log().Info("Schedule crons")
-
-		cronManager := services.NewCronManager(currentScroll.Cronjobs, queueManager)
-		err = cronManager.Init()
+			if err != nil {
+				return err
+			}
 
-		if err != nil {
-			return err
+			logger.Log().Info("Active Scroll",
+				zap.String("Description", fmt.Sprintf("%s (%s)", currentScroll.Desc, currentScroll.Name)),
+				zap.String("Scroll Version", currentScroll.Version.String()),
+				zap.String("cwd", cwd))
 		}
+		err = s.Serve(a, port)
 
-		logger.Log().Info("Active Scroll",
-			zap.String("Description", fmt.Sprintf("%s (%s)", currentScroll.Desc, currentScroll.Name)),
-			zap.String("Scroll Version", currentScroll.Version.String()),
-			zap.String("cwd", cwd))
-
-		s.Serve(a, port)
-
-		return nil
+		return err
 	},
 }
 
@@ -207,7 +205,7 @@ func init() {
 
 	ServeCommand.Flags().StringVarP(&userId, "user-id", "u", "", "Allowed user id")
 
-	ServeCommand.Flags().BoolVarP(&initScroll, "init", "", false, "Initialize the scroll if no lock file is present")
+	ServeCommand.Flags().BoolVarP(&idleScroll, "idle", "", false, "Don't start the queue manager")
 
 	ServeCommand.Flags().BoolVarP(&ignoreVersionCheck, "ignore-version-check", "", false, "Ignore version check")
 

cmd/server/web/server.go

@@ -172,9 +172,11 @@ func (s *Server) SetAPI(app *fiber.App) *fiber.App {
 	return app
 }
 
-func (s *Server) Serve(app *fiber.App, port int) {
+func (s *Server) Serve(app *fiber.App, port int) error {
 	addr := fmt.Sprintf(":%d", port)
 	if err := app.Listen(addr); err != nil {
 		logger.Log().Error("web server error", zap.Error(err))
+		return err
 	}
+	return nil
 }

examples/minecraft/.scroll/scroll.yaml

@@ -20,7 +20,7 @@ commands:
     procedures:
       - mode: stdin
         data:
-          - start
+          - start.0
           - stop
   install:
     run: once

examples/nginx/.scroll/scroll.yaml

@@ -5,7 +5,8 @@ app_version: latest
 init: "start"
 commands:
   start:
-    run: restart
+    # the nginx command detaches the process
+    #run: restart
     procedures:
       - mode: exec
         data: