added retry in connect when vCenter is under stress

Author: alexazarh

package/cloudshell/cp/vcenter/commands/connect_orchestrator.py

@@ -1,3 +1,4 @@
+import traceback
 from multiprocessing.pool import ThreadPool
 
 import jsonpickle
@@ -345,7 +346,8 @@ def _remove_vlan(self, action_mappings, si, vm_uuid, logger):
                 results.append(action_result)
             final_res = self._consolidate_duplicate_results(results)
         except Exception as e:
-            self.logger.error('Exception raised while disconnecting vm({0}) with exception: {1}'.format(vm_uuid, e))
+            self.logger.error('Exception raised while disconnecting vm({0}) with exception: {1}'
+                              .format(vm_uuid, traceback.format_exc()))
             for mode, actions in mode_to_actions.items():
                 for action in actions:
                     error_result = self._create_error_action_res(action, e)

package/cloudshell/cp/vcenter/common/vcenter/vmomi_service.py

@@ -6,6 +6,13 @@
 from cloudshell.cp.vcenter.common.utilites.common_utils import str2bool
 from cloudshell.cp.vcenter.common.vcenter.task_waiter import SynchronousTaskWaiter
 
+
+class VCenterAuthError (Exception):
+    def __init__(self, original_exception):
+        super(VCenterAuthError, self).__init__(original_exception.message)
+        self.original_exception = original_exception
+
+
 class pyVmomiService:
     # region consts
     ChildEntity = 'childEntity'
@@ -61,7 +68,7 @@ def connect(self, address, user, password, port=443):
                 si = self.pyvmomi_connect(host=address, user=user, pwd=password, port=port)
             return si
         except vim.fault.InvalidLogin as e:
-            raise ValueError(e.msg)
+            raise VCenterAuthError(e.msg)
         except IOError as e:
             # logger.info("I/O error({0}): {1}".format(e.errno, e.strerror))
             raise ValueError('Cannot connect to vCenter, please check that the address is valid')

package/cloudshell/cp/vcenter/common/wrappers/command_wrapper.py

@@ -1,7 +1,10 @@
 import inspect
+
+from retrying import retry
+
 from cloudshell.cp.vcenter.common.model_factory import ResourceModelParser
 from cloudshell.cp.vcenter.common.cloud_shell.driver_helper import CloudshellDriverHelper
-from cloudshell.cp.vcenter.common.vcenter.vmomi_service import pyVmomiService
+from cloudshell.cp.vcenter.common.vcenter.vmomi_service import pyVmomiService, VCenterAuthError
 from cloudshell.cp.vcenter.models.VMwarevCenterResourceModel import VMwarevCenterResourceModel
 
 DISCONNCTING_VCENERT = 'disconnecting from vcenter: {0}'
@@ -21,6 +24,10 @@
 LOG_FORMAT = 'action:{0} command_name:{1}'
 
 
+def retry_if_auth_error(ex):
+    return isinstance(ex, VCenterAuthError)
+
+
 class CommandWrapper:
     def __init__(self, pv_service, cloud_shell_helper, resource_model_parser, context_based_logger_factory):
         """
@@ -37,6 +44,7 @@ def __init__(self, pv_service, cloud_shell_helper, resource_model_parser, contex
         self.resource_model_parser = resource_model_parser  # type: ResourceModelParser
         self.context_based_logger_factory = context_based_logger_factory  # type ContextBasedLoggerFactory
 
+    @retry(stop_max_attempt_number=3, wait_fixed=2000, retry_on_exception=retry_if_auth_error)
     def execute_command_with_connection(self, context, command, *args):
         """
         Note: session & vcenter_data_model objects will be injected dynamically to the command
@@ -47,8 +55,8 @@ def execute_command_with_connection(self, context, command, *args):
         """
 
         logger = self.context_based_logger_factory.create_logger_for_context(
-            logger_name='vCenterShell',
-            context=context)
+                logger_name='vCenterShell',
+                context=context)
 
         if not command:
             logger.error(COMMAND_CANNOT_BE_NONE)
@@ -76,9 +84,9 @@ def execute_command_with_connection(self, context, command, *args):
             if connection_details:
                 logger.info(INFO_CONNECTING_TO_VCENTER.format(connection_details.host))
                 logger.debug(
-                    DEBUG_CONNECTION_INFO.format(connection_details.host,
-                                                 connection_details.username,
-                                                 connection_details.port))
+                        DEBUG_CONNECTION_INFO.format(connection_details.host,
+                                                     connection_details.username,
+                                                     connection_details.port))
 
                 si = self.pv_service.connect(connection_details.host,
                                              connection_details.username,

package/requirements.txt

@@ -2,4 +2,5 @@ cloudshell-automation-api>=7.1.0.0,<7.2.0.0
 cloudshell-core>=2.0.0,<2.1.0
 pyvmomi==6.0.0
 jsonpickle==0.9.3
-enum==0.4.6
+enum==0.4.6
+retrying==1.3.3