fixes for agent.send peer disconnect issues during command execution

Author: Glover, Rene (rg9975)

engine/orchestration/src/main/java/com/cloud/agent/manager/AgentManagerImpl.java

@@ -238,6 +238,14 @@ public class AgentManagerImpl extends ManagerBase implements AgentManager, Handl
             false);
     protected final ConfigKey<Integer> RemoteAgentNewConnectionsMonitorInterval = new ConfigKey<>("Advanced", Integer.class, "agent.connections.monitor.interval", "1800",
             "Time in seconds to monitor the new agent connections and cleanup the expired connections.", false);
+
+        protected final ConfigKey<Integer> PeerLookupRetryCount = new ConfigKey<>(Integer.class,
+            "cluster.agent.peer.lookup.retry.count", "Advanced", "1",
+            "Number of retries (in addition to the initial attempt) to resolve the peer management server for a host when forwarding agent commands in a management server cluster.", true);
+
+        protected final ConfigKey<Integer> PeerLookupRetryIntervalMs = new ConfigKey<>(Integer.class,
+            "cluster.agent.peer.lookup.retry.interval.ms", "Advanced", "200",
+            "Sleep interval in milliseconds between peer lookup retries when forwarding agent commands in a management server cluster.", true);
     protected final ConfigKey<Integer> AlertWait = new ConfigKey<>("Advanced", Integer.class, "alert.wait", "1800",
             "Seconds to wait before alerting on a disconnected agent", true);
     protected final ConfigKey<Integer> DirectAgentLoadSize = new ConfigKey<>("Advanced", Integer.class, "direct.agent.load.size", "16",
@@ -790,11 +798,13 @@ public long send(final Long hostId, final Commands commands, final Listener list
     }
 
     protected int getAgentSendRetryCount() {
-        return 0;
+        final String value = _configDao != null ? _configDao.getValue("cluster.agent.peer.lookup.retry.count") : null;
+        return NumbersUtil.parseInt(value, 1);
     }
 
     protected int getAgentSendRetryIntervalMs() {
-        return 0;
+        final String value = _configDao != null ? _configDao.getValue("cluster.agent.peer.lookup.retry.interval.ms") : null;
+        return NumbersUtil.parseInt(value, 200);
     }
 
     protected AgentAttache resolveAttacheForRetry(final Long hostId, final AgentAttache current, final boolean forceReload)
@@ -2271,7 +2281,7 @@ public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[] { CheckTxnBeforeSending, Workers, Port, Wait, AlertWait, DirectAgentLoadSize,
                 DirectAgentPoolSize, DirectAgentThreadCap, EnableKVMAutoEnableDisable, ReadyCommandWait,
                 GranularWaitTimeForCommands, RemoteAgentSslHandshakeTimeout, RemoteAgentMaxConcurrentNewConnections,
-                RemoteAgentNewConnectionsMonitorInterval };
+                RemoteAgentNewConnectionsMonitorInterval, PeerLookupRetryCount, PeerLookupRetryIntervalMs };
     }
 
     protected class SetHostParamsListener implements Listener {

engine/orchestration/src/main/java/com/cloud/agent/manager/ClusteredAgentManagerImpl.java

@@ -162,14 +162,6 @@ protected ClusteredAgentManagerImpl() {
     protected final ConfigKey<Integer> ScanInterval = new ConfigKey<>(Integer.class, "direct.agent.scan.interval", "Advanced", "90", "Interval between scans to load direct agents", false,
             ConfigKey.Scope.Global, 1000);
 
-        protected final ConfigKey<Integer> PeerLookupRetryCount = new ConfigKey<>(Integer.class,
-            "cluster.agent.peer.lookup.retry.count", "Advanced", "0",
-            "Number of retries (in addition to the initial attempt) to resolve the peer management server for a host when forwarding agent commands in a management server cluster.", true);
-
-        protected final ConfigKey<Integer> PeerLookupRetryIntervalMs = new ConfigKey<>(Integer.class,
-            "cluster.agent.peer.lookup.retry.interval.ms", "Advanced", "200",
-            "Sleep interval in milliseconds between peer lookup retries when forwarding agent commands in a management server cluster.", true);
-
     @Override
     public boolean configure(final String name, final Map<String, Object> xmlParams) throws ConfigurationException {
         _peers = new HashMap<>(7);
@@ -1676,8 +1668,6 @@ public ConfigKey<?>[] getConfigKeys() {
         keysLst.add(ConnectedAgentThreshold);
         keysLst.add(LoadSize);
         keysLst.add(ScanInterval);
-        keysLst.add(PeerLookupRetryCount);
-        keysLst.add(PeerLookupRetryIntervalMs);
         return keysLst.toArray(new ConfigKey<?>[keysLst.size()]);
     }
 }