Merge branch 'release-v3.0.0' of https://bitbucket.ngage.netapp.com/scm/sie-bb/netapp-dataops-toolkit into bugfix/blackduck

Author: Karnik, Aarkin

netapp_dataops_traditional/docs/ontap_readme.md

@@ -367,6 +367,8 @@ team1_ws1      300.0GB   5%              285.0GB     15.04GB   2.87GB       flex
 
 The NetApp DataOps Toolkit can be used to mount an existing data volume on your local host. The command for mounting an existing volume locally is `netapp_dataops_cli.py mount volume`. If executed on a Linux host, this command must be run as root. It is usually not necessary to run this command as root on macOS hosts.
 
+**Note:** When mounting volumes, the toolkit must be configured for the user executing the mount command. If running as root (using `sudo`), run `sudo netapp_dataops_cli.py config` to create a configuration file for the root user before attempting to mount volumes.
+
 The following options/arguments are required:
 
 ```
@@ -401,6 +403,8 @@ Volume mounted successfully.
 
 The NetApp DataOps Toolkit can be used to unmount an existing data volume that is currently on your local host. The command for unmounting an existing volume is `netapp_dataops_cli.py unmount volume`. If executed on a Linux host, this command must be run as root. It is usually not necessary to run this command as root on macOS hosts.
 
+**Note:** When unmounting volumes, the toolkit must be configured for the user executing the unmount command. If running as root (using `sudo`), run `sudo netapp_dataops_cli.py config` to create a configuration file for the root user before attempting to unmount volumes.
+
 The following options/arguments are required:
 
 ```
@@ -1663,6 +1667,9 @@ APIConnectionError              # The storage system/service API returned an err
 #### Mount an Existing Data Volume Locally
 
 The NetApp DataOps Toolkit can be used to mount an existing data volume as "read-only" or "read-write" on your local host as part of any Python program or workflow. On Linux hosts, mounting requires root privileges, so any Python program that invokes this function must be run as root. It is usually not necessary to invoke this function as root on macOS hosts.
+
+**Note:** When mounting volumes programmatically, the toolkit must be configured for the user executing the program. If the program runs as root (using `sudo`), run `sudo netapp_dataops_cli.py config` to create a configuration file for the root user before attempting to mount volumes.
+
 ##### Function Definition
 
 ```py
@@ -1698,6 +1705,8 @@ MountOperationError             # The volume was not succesfully mounted locally
 
 The NetApp DataOps Toolkit can be used to near-instantaneously unmount an existing data volume (that is currently mounted on your local host) as part of any Python program or workflow.
 
+**Note:** When unmounting volumes programmatically, the toolkit must be configured for the user executing the program. If the program runs as root (using `sudo`), run `sudo netapp_dataops_cli.py config` to create a configuration file for the root user before attempting to unmount volumes.
+
 ##### Function Definition
 
 ```py

netapp_dataops_traditional/netapp_dataops/commands/create_command.py

@@ -373,7 +373,7 @@ def _create_cifs_share(self) -> None:
         try:
             opts, _ = getopt.getopt(
                 self.args[3:], 
-                "u:h:s:n:l:a:v:c:", 
+                "u:hs:n:l:a:v:c:", 
                 ["cluster-name=", "help", "svm=", "name=", "properties=", "acls=", "volume=", "comment="]
             )
         except Exception as err:

netapp_dataops_traditional/netapp_dataops/commands/get_command.py

@@ -55,7 +55,7 @@ def _get_cifs_share(self) -> None:
         try:
             opts, args = getopt.getopt(
                 self.args[3:], 
-                "u:h:s:n:", 
+                "u:hs:n:", 
                 ["cluster-name=", "help", "svm=", "name="]
             )
         except Exception as err:

netapp_dataops_traditional/netapp_dataops/config/dataset_manager.py

@@ -167,8 +167,9 @@ def _setup_existing_root_volume(self, config: DatasetManagerConfig) -> None:
             volume_info = self._get_volume_info(root_volume_name)
 
             if volume_info is None:
-                logger.info(f"  Warning: Volume '{root_volume_name}' not found on ONTAP.")
-                logger.info("  Configuration has been saved. Please verify the volume name and try setup again.")
+                logger.info(f"  Note: Cannot validate volume '{root_volume_name}' on ONTAP at this time.")
+                logger.info("  Configuration has been saved. Volume validation will occur when you use Dataset Manager.")
+                # Skip mounting setup since we can't validate the volume
                 return
 
             # Check junction path
@@ -197,12 +198,39 @@ def _setup_existing_root_volume(self, config: DatasetManagerConfig) -> None:
         self._handle_root_volume_mounting(root_volume_name, root_mountpoint)
 
     def _setup_new_root_volume(self, config: DatasetManagerConfig) -> None:
-        """Perform operations for new root volume after config is saved."""
+        """Perform operations for new root volume after config is saved.
+        
+        Note: This method assumes the base ONTAP configuration has already been
+        saved to disk before being called, as it needs to access ONTAP APIs.
+        """
         root_volume_name = config.root_volume_name
         root_mountpoint = config.root_mountpoint
 
         logger.info(f"\n  Setting up new Dataset Manager root volume '{root_volume_name}'...")
 
+        # Verify we can connect to ONTAP before proceeding
+        try:
+            from ..traditional.core import _retrieve_config
+            from ..traditional.exceptions import InvalidConfigError
+            
+            # Try to retrieve config to verify it exists and is valid
+            try:
+                ontap_config = _retrieve_config(print_output=False)
+                # Verify we have the minimum required fields
+                if not all(k in ontap_config for k in ['hostname', 'svm', 'dataLif']):
+                    raise InvalidConfigError("Missing required ONTAP configuration fields")
+            except (InvalidConfigError, FileNotFoundError, KeyError) as e:
+                logger.info(f"  Note: Cannot create root volume - ONTAP configuration not available yet.")
+                logger.info(f"  Dataset Manager configuration has been saved.")
+                logger.info(f"  The root volume will be created when you first use Dataset Manager,")
+                logger.info(f"  or you can create it manually: netapp_dataops_cli.py create volume -n {root_volume_name} -s 1GB")
+                return
+        except Exception as e:
+            logger.info(f"  Note: Cannot validate ONTAP connection: {e}")
+            logger.info(f"  Dataset Manager configuration has been saved.")
+            logger.info(f"  Please create the root volume manually or reconfigure later.")
+            return
+        
         while True:  # Loop for retrying with different names if needed
             try:
                 # Check if volume already exists
@@ -250,7 +278,10 @@ def _setup_new_root_volume(self, config: DatasetManagerConfig) -> None:
         self._handle_root_volume_mounting(root_volume_name, root_mountpoint)
 
     def _get_volume_info(self, volume_name: str) -> Optional[Dict[str, Any]]:
-        """Get volume information from ONTAP."""
+        """Get volume information from ONTAP.
+        
+        Returns None if ONTAP connection cannot be established or volume not found.
+        """
         try:
             # Use list_volumes to find the specific volume
             # Always suppress output to avoid displaying volume lists during user input
@@ -260,12 +291,17 @@ def _get_volume_info(self, volume_name: str) -> Optional[Dict[str, Any]]:
                     return volume
             return None
         except Exception as e:
+            # Could be a connection error or other issue
+            # Return None to allow graceful handling by caller
             if self.print_output:
-                logger.info(f"Error retrieving volume information: {e}")
+                logger.info(f"  Note: Could not retrieve volume information: {e}")
             return None
 
     def _junction_path_exists(self, junction_path: str, exclude_volume: str = None) -> bool:
-        """Check if a junction path is already in use by any volume."""
+        """Check if a junction path is already in use by any volume.
+        
+        Returns False if ONTAP connection cannot be established or on error.
+        """
         try:
             # Always suppress output to avoid displaying volume lists during validation
             volumes = volume_operations.list_volumes(print_output=False)
@@ -308,36 +344,31 @@ def _create_root_volume_on_ontap(self, volume_name: str) -> bool:
             return False
 
     def _is_mounted(self, mountpoint: str) -> bool:
-        """Check if a mountpoint is currently in use."""
+        """Check if a mountpoint is currently in use (cross-platform)."""
         try:
-            # Check if mountpoint utility is available
-            self._check_required_utilities('mountpoint')
-            
-            result = subprocess.run(['mountpoint', '-q', mountpoint], 
-                                  capture_output=True, text=True)
-            return result.returncode == 0
-        except RuntimeError as e:
-            # Re-raise utility check errors
-            raise
+            # Use mount command which is available on both Linux and macOS
+            result = subprocess.run(['mount'], capture_output=True, text=True)
+            if result.returncode == 0:
+                # Check if mountpoint appears in mount output
+                for line in result.stdout.split('\n'):
+                    # Match " on <mountpoint> " pattern to avoid false positives
+                    if f" on {mountpoint} " in line or line.endswith(f" on {mountpoint}"):
+                        return True
+            return False
         except Exception:
             return False
 
     def _get_mount_target(self, mountpoint: str) -> str:
-        """Get what is currently mounted at the given mountpoint."""
+        """Get what is currently mounted at the given mountpoint (cross-platform)."""
         try:
-            # Check if mount utility is available
-            self._check_required_utilities('mount')
-            
             result = subprocess.run(['mount'], capture_output=True, text=True)
             if result.returncode == 0:
                 for line in result.stdout.split('\n'):
-                    if f" {mountpoint} " in line:
+                    # Match " on <mountpoint> " pattern
+                    if f" on {mountpoint} " in line or line.endswith(f" on {mountpoint}"):
                         # Extract the source (first part before " on ")
                         return line.split(' on ')[0]
             return "unknown"
-        except RuntimeError as e:
-            # Re-raise utility check errors
-            raise
         except Exception:
             return "unknown"
 
@@ -365,6 +396,8 @@ def _handle_root_volume_mounting(self, volume_name: str, mountpoint: str) -> Non
             # Add to fstab FIRST, then mount using fstab entry
             if self._add_to_fstab(volume_name, mountpoint, expected_nfs_target):
                 logger.info(f"  Volume '{volume_name}' added to fstab")
+                # Now mount the volume using the fstab entry
+                self._mount_from_fstab(mountpoint)
 
     def _add_to_fstab(self, volume_name: str, mountpoint: str, nfs_target: str) -> bool:
         """Add volume to fstab first, then mount using fstab entry (following requirements)."""
@@ -392,6 +425,43 @@ def _add_to_fstab(self, volume_name: str, mountpoint: str, nfs_target: str) -> b
             logger.info(f"  Error in fstab setup and mount: {e}")
             return False
 
+    def _mount_from_fstab(self, mountpoint: str) -> bool:
+        """
+        Mount a volume using its fstab entry.
+        
+        Args:
+            mountpoint: The mountpoint path to mount
+            
+        Returns:
+            bool: True if mount successful, False otherwise
+        """
+        try:
+            logger.info(f"  Mounting volume at '{mountpoint}'...")
+            
+            # Check if we need sudo
+            needs_sudo = not os.access('/etc', os.W_OK)
+            
+            if needs_sudo:
+                result = subprocess.run(['sudo', 'mount', mountpoint],
+                                      capture_output=True, text=True)
+            else:
+                result = subprocess.run(['mount', mountpoint],
+                                      capture_output=True, text=True)
+            
+            if result.returncode == 0:
+                logger.info(f"  Successfully mounted volume at '{mountpoint}'")
+                return True
+            else:
+                error_msg = result.stderr.strip()
+                logger.info(f"  Warning: Failed to mount volume: {error_msg}")
+                logger.info(f"  You can manually mount it later with: sudo mount {mountpoint}")
+                return False
+                
+        except Exception as e:
+            logger.info(f"  Warning: Error mounting volume: {e}")
+            logger.info(f"  You can manually mount it later with: sudo mount {mountpoint}")
+            return False
+    
     def _get_expected_nfs_target(self, volume_name: str) -> str:
         """Get the expected NFS target for a volume."""
         try:

netapp_dataops_traditional/netapp_dataops/config/manager.py

@@ -126,16 +126,35 @@ def create_interactive_config(self) -> NetAppDataOpsConfig:
             dataset_manager_config = None
             if PromptUtils.prompt_yes_no("Do you intend to use the toolkit's Dataset Manager functionality?"):
                 logger.info("\nDataset Manager Configuration:")
+                
+                # IMPORTANT: Save base ONTAP config to disk BEFORE Dataset Manager setup
+                # This allows Dataset Manager to connect to ONTAP and create volumes
+                base_config = NetAppDataOpsConfig(
+                    ontap=ontap_config,
+                    s3=s3_config,
+                    cloud_sync=cloud_sync_config,
+                    dataset_manager=None  # Will be added after setup
+                )
+                self.save_config(base_config)
+                
+                # Now Dataset Manager can connect to ONTAP using the saved config
                 dataset_manager_configurator = DatasetManagerConfigurator()
                 dataset_manager_config = dataset_manager_configurator.configure_dataset_manager()
 
-            return NetAppDataOpsConfig(
+            # Create final configuration object
+            final_config = NetAppDataOpsConfig(
                 ontap=ontap_config,
                 s3=s3_config,
                 cloud_sync=cloud_sync_config,
                 dataset_manager=dataset_manager_config
             )
 
+            # Display configuration summary
+            summary = self.get_config_summary(final_config)
+            logger.info(summary)
+            
+            return final_config
+            
         except Exception as e:
             raise ConfigCreationError(f"Failed to create configuration: {e}")
 
@@ -338,6 +357,7 @@ def get_config_summary(self, config: NetAppDataOpsConfig) -> str:
             str: Configuration summary
         """
         summary = []
+        summary.append(f"\nCreated config file: {self.config_file}")
         summary.append("\nConfiguration Summary:")
         summary.append(f"  ONTAP Host: {config.ontap.hostname}")
         summary.append(f"  SVM: {config.ontap.svm}")

netapp_dataops_traditional/netapp_dataops/traditional/data_movement/s3_operations.py

@@ -19,49 +19,12 @@
     _print_invalid_config_error,
     deprecated
 )
+from ..core.connection import _instantiate_s3_session
+from ..core.config import _retrieve_s3_access_details
 
 logger = setup_logger(__name__)
 
 
-def _instantiate_s3_session(s3Endpoint: str, s3AccessKeyId: str, s3SecretAccessKey: str, s3VerifySSLCert: bool, s3CACertBundle: str, print_output: bool = False):
-    # Instantiate session
-    session = boto3.session.Session(aws_access_key_id=s3AccessKeyId, aws_secret_access_key=s3SecretAccessKey)
-    config = BotoConfig(signature_version='s3v4')
-
-    if s3VerifySSLCert:
-        if s3CACertBundle:
-            s3 = session.resource(service_name='s3', endpoint_url=s3Endpoint, verify=s3CACertBundle, config=config)
-        else:
-            s3 = session.resource(service_name='s3', endpoint_url=s3Endpoint, config=config)
-    else:
-        s3 = session.resource(service_name='s3', endpoint_url=s3Endpoint, verify=False, config=config)
-
-    return s3
-
-
-def _retrieve_s3_access_details(print_output: bool = False) -> Tuple[str, str, str, bool, str]:
-    try:
-        config = _retrieve_config(print_output=print_output)
-    except InvalidConfigError:
-        raise
-    try:
-        s3Endpoint = config["s3Endpoint"]
-        s3AccessKeyId = config["s3AccessKeyId"]
-        s3SecretAccessKeyBase64 = config["s3SecretAccessKey"]
-        s3VerifySSLCert = config["s3VerifySSLCert"]
-        s3CACertBundle = config["s3CACertBundle"]
-    except KeyError:
-        if print_output:
-            _print_invalid_config_error()
-        raise InvalidConfigError()
-
-    s3SecretAccessKeyBase64Bytes = s3SecretAccessKeyBase64.encode("ascii")
-    s3SecretAccessKeyBytes = base64.b64decode(s3SecretAccessKeyBase64Bytes)
-    s3SecretAccessKey = s3SecretAccessKeyBytes.decode("ascii")
-
-    return s3Endpoint, s3AccessKeyId, s3SecretAccessKey, s3VerifySSLCert, s3CACertBundle
-
-
 def _download_from_s3(s3Endpoint: str, s3AccessKeyId: str, s3SecretAccessKey: str, s3VerifySSLCert: bool,
                    s3CACertBundle: str, s3Bucket: str, s3ObjectKey: str, localFile: str, print_output: bool = False):
     # Instantiate S3 session