Add spawnpoint health check before performing client operations

Author: jhkolb

spawnclient/client.go

@@ -16,11 +16,11 @@ import (
 	"github.com/pkg/errors"
 )
 
-type SpawnClient struct {
+type Client struct {
 	bwClient *bw2.BW2Client
 }
 
-func New(router, entityFile string) (*SpawnClient, error) {
+func New(router, entityFile string) (*Client, error) {
 	bw2.SilenceLog()
 	client, err := bw2.Connect(router)
 	if err != nil {
@@ -30,10 +30,18 @@ func New(router, entityFile string) (*SpawnClient, error) {
 		return nil, errors.Wrap(err, "Failed to set BW2 entity file")
 	}
 
-	return &SpawnClient{bwClient: client}, nil
+	return &Client{bwClient: client}, nil
 }
 
-func (sc *SpawnClient) Scan(baseURI string) (map[string]daemon.Heartbeat, error) {
+func (sc *Client) Scan(baseURI string) (map[string]daemon.Heartbeat, error) {
+	namespace := strings.Split(baseURI, "/")[0]
+	vk, _, err := sc.bwClient.ResolveLongAlias(namespace)
+	aliased := (err == nil)
+	var encodedVk string
+	if aliased {
+		encodedVk = base64.URLEncoding.EncodeToString(vk)
+	}
+
 	svcClient := sc.bwClient.NewServiceClient(baseURI, "s.spawnpoint")
 	iFaceClient := svcClient.AddInterface("daemon", "i.spawnpoint")
 	heartbeatMsgs, err := sc.bwClient.Query(&bw2.QueryParams{
@@ -52,16 +60,21 @@ func (sc *SpawnClient) Scan(baseURI string) (map[string]daemon.Heartbeat, error)
 					// Ignore this query result
 					continue
 				}
-				uri := msg.URI[:len(msg.URI)-len("/s.spawnpoint/daemon/i.spawnpoint/signal/heartbeat")]
-				spawnpoints[uri] = hb
+				rawURI := msg.URI[:len(msg.URI)-len("/s.spawnpoint/daemon/i.spawnpoint/signal/heartbeat")]
+				if aliased {
+					convertedURI := strings.Replace(rawURI, encodedVk, namespace, 1)
+					spawnpoints[convertedURI] = hb
+				} else {
+					spawnpoints[rawURI] = hb
+				}
 			}
 		}
 	}
 
 	return spawnpoints, nil
 }
 
-func (sc *SpawnClient) Inspect(uri string) (*daemon.Heartbeat, map[string]daemon.ServiceHeartbeat, error) {
+func (sc *Client) Inspect(uri string) (*daemon.Heartbeat, map[string]daemon.ServiceHeartbeat, error) {
 	daemonHbs, err := sc.Scan(uri)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "Initial spawnpoint scan failed")
@@ -103,7 +116,7 @@ func (sc *SpawnClient) Inspect(uri string) (*daemon.Heartbeat, map[string]daemon
 	return &daemonHb, svcHeartbeats, nil
 }
 
-func (sc *SpawnClient) Deploy(config *service.Configuration, uri string) error {
+func (sc *Client) Deploy(config *service.Configuration, uri string) error {
 	if err := validateConfig(config); err != nil {
 		return errors.Wrap(err, "Invalid service configuration")
 	}
@@ -135,7 +148,7 @@ func (sc *SpawnClient) Deploy(config *service.Configuration, uri string) error {
 	return nil
 }
 
-func (sc *SpawnClient) Stop(uri string, svcName string) error {
+func (sc *Client) Stop(uri string, svcName string) error {
 	svcClient := sc.bwClient.NewServiceClient(uri, "s.spawnpoint")
 	iFaceClient := svcClient.AddInterface(svcName, "i.spawnable")
 	if err := iFaceClient.PublishSlot("stop"); err != nil {
@@ -144,7 +157,7 @@ func (sc *SpawnClient) Stop(uri string, svcName string) error {
 	return nil
 }
 
-func (sc *SpawnClient) Restart(uri string, svcName string) error {
+func (sc *Client) Restart(uri string, svcName string) error {
 	svcClient := sc.bwClient.NewServiceClient(uri, "s.spawnpoint")
 	iFaceClient := svcClient.AddInterface(svcName, "i.spawnable")
 	if err := iFaceClient.PublishSlot("restart"); err != nil {
@@ -153,7 +166,7 @@ func (sc *SpawnClient) Restart(uri string, svcName string) error {
 	return nil
 }
 
-func (sc *SpawnClient) Tail(ctx context.Context, svcName string, uri string) (<-chan service.LogMessage, <-chan error) {
+func (sc *Client) Tail(ctx context.Context, svcName string, uri string) (<-chan service.LogMessage, <-chan error) {
 	svcClient := sc.bwClient.NewServiceClient(uri, "s.spawnpoint")
 	iFaceClient := svcClient.AddInterface(svcName, "i.spawnable")
 	errChan := make(chan error, 1)

spawnctl/main.go

@@ -18,6 +18,7 @@ import (
 )
 
 const deploymentHistVar = "SPAWNCTL_HISTORY_FILE"
+const healthHorizon = 5 * time.Minute
 
 func main() {
 	app := cli.NewApp()
@@ -226,6 +227,19 @@ func actionDeploy(c *cli.Context) error {
 	spawnClient, err := spawnclient.New(c.GlobalString("router"), entity)
 	if err != nil {
 		fmt.Printf("Could not create spawnpoint client: %s\n", err)
+		os.Exit(1)
+	}
+	age, err := checkSpawnpointHealth(spawnClient, spawnpointURI)
+	if err != nil {
+		fmt.Printf("Failed to check spawnpoint health: %s\n", err)
+		os.Exit(1)
+	} else if age == 0 {
+		fmt.Printf("No spawnpoint exists at %s\n", spawnpointURI)
+		os.Exit(1)
+	} else if age > healthHorizon {
+		fmt.Printf("Spawnpoint at %s appears to be down\n", spawnpointURI)
+		fmt.Printf("Last seen %s ago\n", age.String())
+		os.Exit(1)
 	}
 
 	var ctx context.Context
@@ -316,6 +330,19 @@ func manipulateService(c *cli.Context, command string) error {
 	spawnClient, err := spawnclient.New(c.GlobalString("router"), entity)
 	if err != nil {
 		fmt.Printf("Could not create spawnpoint client: %s\n", err)
+		os.Exit(1)
+	}
+	age, err := checkSpawnpointHealth(spawnClient, spawnpointURI)
+	if err != nil {
+		fmt.Printf("Failed to check spawnpoint health: %s\n", err)
+		os.Exit(1)
+	} else if age == 0 {
+		fmt.Printf("No spawnpoint exists at %s\n", spawnpointURI)
+		os.Exit(1)
+	} else if age > healthHorizon {
+		fmt.Printf("Spawnpoint at %s appears to be down\n", spawnpointURI)
+		fmt.Printf("Last seen %s ago\n", age.String())
+		os.Exit(1)
 	}
 
 	var ctx context.Context
@@ -448,3 +475,18 @@ func parseSvcConfig(configFile string) (*service.Configuration, error) {
 
 	return &svcConfig, nil
 }
+
+func checkSpawnpointHealth(sc *spawnclient.Client, uri string) (time.Duration, error) {
+	spawnpoints, err := sc.Scan(uri)
+	if err != nil {
+		return 0, errors.Wrap(err, "Failed to scan for spawnpoint at URI")
+	}
+	daemonHb, ok := spawnpoints[uri]
+	if !ok {
+		return 0, nil
+	}
+
+	daemonTimestamp := time.Unix(0, daemonHb.Time)
+	duration := time.Now().Sub(daemonTimestamp)
+	return duration, nil
+}