retry on failure (#10)

Author: joecorall

.github/workflows/lint-test-build-push.yml

@@ -1,6 +1,7 @@
 name: build-push
 on:
   push:
+  workflow_dispatch:
 jobs:
   test:
     permissions:

main.go

@@ -164,6 +164,11 @@ func main() {
 		os.Exit(0)
 	}
 
+	retryDelay := time.Second
+	const maxRetryDelay = time.Minute
+	retryAttempts := 0
+	const maxRetryAttempts = 10
+
 	for {
 		select {
 		case <-signalCh:
@@ -173,6 +178,23 @@ func main() {
 		request, err := newVaultRequest(http.MethodHead, vaultAddr+"/v1/sys/health", nil)
 		if err != nil {
 			log.Println(err)
+			if checkInterval < 0 {
+				retryAttempts++
+				if retryAttempts >= maxRetryAttempts {
+					log.Printf("Health check failed after %d attempts; exiting with failure", retryAttempts)
+					os.Exit(1)
+				}
+				log.Printf(
+					"Retrying health check in %s (exponential backoff), attempt %d/%d",
+					retryDelay, retryAttempts, maxRetryAttempts,
+				)
+				time.Sleep(retryDelay)
+				retryDelay *= 2
+				if retryDelay > maxRetryDelay {
+					retryDelay = maxRetryDelay
+				}
+				continue
+			}
 			time.Sleep(checkInterval)
 			continue
 		}
@@ -185,10 +207,30 @@ func main() {
 
 		if err != nil {
 			log.Println(err)
+			if checkInterval < 0 {
+				retryAttempts++
+				if retryAttempts >= maxRetryAttempts {
+					log.Printf("Health check failed after %d attempts; exiting with failure", retryAttempts)
+					os.Exit(1)
+				}
+				log.Printf(
+					"Retrying health check in %s (exponential backoff), attempt %d/%d",
+					retryDelay, retryAttempts, maxRetryAttempts,
+				)
+				time.Sleep(retryDelay)
+				retryDelay *= 2
+				if retryDelay > maxRetryDelay {
+					retryDelay = maxRetryDelay
+				}
+				continue
+			}
 			time.Sleep(checkInterval)
 			continue
 		}
 
+		retryDelay = time.Second
+		retryAttempts = 0
+
 		switch response.StatusCode {
 		case 200:
 			log.Println("Vault is initialized and unsealed.")