Skip to content

Commit 4af5139

Browse files
committed
[core] Ensure AutoEnv SOR failure goes to ERROR
1 parent 78894d7 commit 4af5139

1 file changed

Lines changed: 39 additions & 0 deletions

File tree

core/environment/manager.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -937,6 +937,7 @@ func (envs *Manager) CreateAutoEnvironment(workflowPath string, userVars map[str
937937
log.WithField("partition", env.Id().String()).
938938
WithField("state", envState).
939939
Debug("could not transition to ERROR after failed deployment/configuration, cleanup in progress")
940+
940941
}
941942

942943
envTasks := env.Workflow().GetTasks()
@@ -973,7 +974,45 @@ func (envs *Manager) CreateAutoEnvironment(workflowPath string, userVars map[str
973974

974975
err = env.TryTransition(trans)
975976
if err != nil {
977+
envState := env.CurrentState()
978+
976979
env.sendEnvironmentEvent(&event.EnvironmentEvent{EnvironmentID: env.Id().String(), Error: err})
980+
981+
log.WithField("state", envState).
982+
WithField("environment", env.Id().String()).
983+
WithError(err).
984+
Warn("environment start activity failed, cleanup in progress")
985+
986+
err = env.TryTransition(NewGoErrorTransition(
987+
envs.taskman),
988+
)
989+
if err != nil {
990+
log.WithField("partition", env.Id().String()).
991+
WithField("state", envState).
992+
Debug("could not transition to ERROR after failed start activity, cleanup in progress")
993+
994+
env.setState("ERROR")
995+
}
996+
997+
envTasks := env.Workflow().GetTasks()
998+
// TeardownEnvironment manages the envs.mu internally
999+
err = envs.TeardownEnvironment(env.Id(), true /*force*/)
1000+
if err != nil {
1001+
env.sendEnvironmentEvent(&event.EnvironmentEvent{EnvironmentID: env.Id().String(), Error: err})
1002+
}
1003+
1004+
killedTasks, _, rlsErr := envs.taskman.KillTasks(envTasks.GetTaskIds())
1005+
if rlsErr != nil {
1006+
log.WithError(rlsErr).Warn("task teardown error")
1007+
}
1008+
log.WithFields(logrus.Fields{
1009+
"killedCount": len(killedTasks),
1010+
"lastEnvState": envState,
1011+
"level": infologger.IL_Support,
1012+
"partition": env.Id().String(),
1013+
}).Info("environment start activity failed, tasks were cleaned up")
1014+
1015+
log.WithField("partition", env.Id().String()).Info("environment teardown complete")
9771016
return
9781017
}
9791018

0 commit comments

Comments
 (0)