Add support for creating environments with existing python virtual environments (#257)

* Add support for creating environments with existing python virtual environments

* Update tests

* Add tests for external deepnote environments, other minor changes

- Added timeout to Python execution for virtual environment detection.
- Implemented tests for renaming, deleting, and switching between managed and external environments.
- Updated package management for external environments.
- Improved error messages for missing deepnote-toolkit installation.

* Implement installation of deepnote-toolkit in existing virtual environments

- Added a new method `installToolkitInExistingVenv` to the `DeepnoteToolkitInstaller` class for installing the toolkit in an existing external virtual environment.
- Updated the `IDeepnoteToolkitInstaller` interface to include the new method.
- Enhanced error handling and logging during the installation process.
- Modified the `handleKernelSelectionError` method to provide user-friendly actions for missing toolkit installations.
- Introduced a new private method `installToolkitAndNotify` in `DeepnoteKernelAutoSelector` to handle toolkit installation and notify users upon completion.

* Enhance Deepnote toolkit installation handling

- Added logic to wait for any pending installations in the `DeepnoteToolkitInstaller` class, improving user experience during concurrent installation requests.
- Updated the error message in `DeepnoteKernelAutoSelector` to inform users about the required deepnote-toolkit version for running Deepnote projects, enhancing clarity and guidance for users.

* Properly track toolkit installation promises

* Remove commented out code

Author: tkislan

src/kernels/deepnote/deepnoteServerStarter.node.ts

@@ -95,13 +95,15 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
      * Environment-based method: Start a server for a kernel environment.
      * @param interpreter The Python interpreter to use
      * @param venvPath The path to the venv
+     * @param managedVenv Whether the venv is managed by this extension (created by us)
      * @param environmentId The environment ID (used as key for server management)
      * @param token Cancellation token
      * @returns Server connection information
      */
     public async startServer(
         interpreter: PythonEnvironment,
         venvPath: Uri,
+        managedVenv: boolean,
         additionalPackages: string[],
         environmentId: string,
         deepnoteFileUri: Uri,
@@ -165,6 +167,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
                 existingContext,
                 interpreter,
                 venvPath,
+                managedVenv,
                 additionalPackages,
                 environmentId,
                 deepnoteFileUri,
@@ -235,6 +238,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
         projectContext: ProjectContext,
         interpreter: PythonEnvironment,
         venvPath: Uri,
+        managedVenv: boolean,
         additionalPackages: string[],
         environmentId: string,
         deepnoteFileUri: Uri,
@@ -250,6 +254,7 @@ export class DeepnoteServerStarter implements IDeepnoteServerStarter, IExtension
         const { pythonInterpreter: venvInterpreter } = await this.toolkitInstaller.ensureVenvAndToolkit(
             interpreter,
             venvPath,
+            managedVenv,
             token
         );
 

src/kernels/deepnote/deepnoteToolkitInstaller.node.ts

@@ -4,15 +4,19 @@
 import { inject, injectable, named } from 'inversify';
 import { CancellationToken, l10n, Uri, workspace } from 'vscode';
 
+import { Cancellation } from '../../platform/common/cancellation';
 import { STANDARD_OUTPUT_CHANNEL } from '../../platform/common/constants';
 import { IFileSystem } from '../../platform/common/platform/types';
 import { IProcessServiceFactory } from '../../platform/common/process/types.node';
 import { IExtensionContext, IOutputChannel } from '../../platform/common/types';
-import { DeepnoteToolkitInstallError, DeepnoteVenvCreationError } from '../../platform/errors/deepnoteKernelErrors';
+import {
+    DeepnoteToolkitInstallError,
+    DeepnoteToolkitMissingError,
+    DeepnoteVenvCreationError
+} from '../../platform/errors/deepnoteKernelErrors';
 import { logger } from '../../platform/logging';
 import { PythonEnvironment } from '../../platform/pythonEnvironments/info';
 import { DEEPNOTE_TOOLKIT_VERSION, IDeepnoteToolkitInstaller, VenvAndToolkitInstallation } from './types';
-import { Cancellation } from '../../platform/common/cancellation';
 
 /**
  * Handles installation of the deepnote-toolkit Python package.
@@ -77,6 +81,7 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
     public async ensureVenvAndToolkit(
         baseInterpreter: PythonEnvironment,
         venvPath: Uri,
+        managedVenv: boolean,
         token?: CancellationToken
     ): Promise<VenvAndToolkitInstallation> {
         const venvKey = venvPath.fsPath;
@@ -86,7 +91,7 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
 
         // Validate that venv path is in current globalStorage (not from a different editor like VS Code)
         const expectedStoragePrefix = this.context.globalStorageUri.fsPath;
-        if (!venvKey.startsWith(expectedStoragePrefix)) {
+        if (managedVenv && !venvKey.startsWith(expectedStoragePrefix)) {
             const error = new Error(
                 `Venv path mismatch! Expected venv under ${expectedStoragePrefix} but got ${venvKey}. ` +
                     `This might happen if the notebook was previously used in a different editor (VS Code vs Cursor).`
@@ -139,6 +144,10 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
             }
         }
 
+        if (!managedVenv) {
+            throw new DeepnoteToolkitMissingError(baseInterpreter.uri.fsPath, venvPath.fsPath);
+        }
+
         // Start the installation and track it
         const installation = this.installVenvAndToolkit(baseInterpreter, venvPath, token);
         this.pendingInstallations.set(venvKey, installation);
@@ -203,6 +212,46 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
         }
     }
 
+    /**
+     * Install deepnote-toolkit in an existing external venv.
+     * This is used when the user has an external venv without toolkit installed.
+     * @param venvPath Path to the existing venv
+     * @param token Cancellation token
+     * @returns The venv Python interpreter and toolkit version if successful
+     */
+    public async installToolkitInExistingVenv(
+        venvPath: Uri,
+        token?: CancellationToken
+    ): Promise<VenvAndToolkitInstallation> {
+        const venvKey = venvPath.fsPath;
+
+        // Wait for any pending installation
+        const pendingInstall = this.pendingInstallations.get(venvKey);
+        if (pendingInstall) {
+            logger.info(`Waiting for pending installation for ${venvKey}...`);
+            return pendingInstall;
+        }
+
+        const venvInterpreter = await this.getVenvInterpreterByPath(venvPath);
+        if (!venvInterpreter) {
+            throw new Error(`Venv not found at ${venvPath.fsPath}`);
+        }
+
+        logger.info(`Installing deepnote-toolkit in existing venv at ${venvPath.fsPath}`);
+        this.outputChannel.appendLine(l10n.t('Installing deepnote-toolkit in existing environment...'));
+
+        const installation = this.installToolkitPackages(venvInterpreter, venvPath, token);
+        this.pendingInstallations.set(venvKey, installation);
+
+        try {
+            return await installation;
+        } finally {
+            if (this.pendingInstallations.get(venvKey) === installation) {
+                this.pendingInstallations.delete(venvKey);
+            }
+        }
+    }
+
     /**
      * Install venv and toolkit at a specific path (environment-based).
      */
@@ -257,101 +306,114 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
                 );
             }
 
-            // Use undefined as resource to get full system environment (including git in PATH)
-            const venvProcessService = await this.processServiceFactory.create(undefined);
+            // Use the shared helper method to install toolkit packages
+            return await this.installToolkitPackages(venvInterpreter, venvPath, token);
+        } catch (ex) {
+            // If this is already a DeepnoteKernelError, rethrow it without wrapping
+            if (ex instanceof DeepnoteVenvCreationError || ex instanceof DeepnoteToolkitInstallError) {
+                throw ex;
+            }
 
-            // Upgrade pip in the venv to the latest version
-            logger.info('Upgrading pip in venv to latest version...');
-            this.outputChannel.appendLine(l10n.t('Upgrading pip...'));
-            const pipUpgradeResult = await venvProcessService.exec(
-                venvInterpreter.uri.fsPath,
-                ['-m', 'pip', 'install', '--upgrade', 'pip'],
-                { throwOnStdErr: false }
+            // Otherwise, log full details and wrap in a generic toolkit install error
+            logger.error('Failed to set up deepnote-toolkit', ex);
+            this.outputChannel.appendLine(l10n.t('Failed to set up deepnote-toolkit; see logs for details'));
+
+            throw new DeepnoteToolkitInstallError(
+                baseInterpreter.uri.fsPath,
+                venvPath.fsPath,
+                DEEPNOTE_TOOLKIT_VERSION,
+                '',
+                ex instanceof Error ? ex.message : String(ex),
+                ex instanceof Error ? ex : undefined
             );
+        }
+    }
 
-            if (pipUpgradeResult.stdout) {
-                logger.info(`pip upgrade output: ${pipUpgradeResult.stdout}`);
-            }
-            if (pipUpgradeResult.stderr) {
-                logger.info('pip upgrade stderr', pipUpgradeResult.stderr);
-            }
+    /**
+     * Shared helper to install toolkit packages in a venv.
+     * Used by both installVenvAndToolkit (managed venvs) and installToolkitInExistingVenv (external venvs).
+     */
+    private async installToolkitPackages(
+        venvInterpreter: PythonEnvironment,
+        venvPath: Uri,
+        token?: CancellationToken
+    ): Promise<VenvAndToolkitInstallation> {
+        // Use undefined as resource to get full system environment (including git in PATH)
+        const venvProcessService = await this.processServiceFactory.create(undefined);
 
-            Cancellation.throwIfCanceled(token);
+        // Upgrade pip in the venv to the latest version
+        logger.info('Upgrading pip in venv to latest version...');
+        this.outputChannel.appendLine(l10n.t('Upgrading pip...'));
+        const pipUpgradeResult = await venvProcessService.exec(
+            venvInterpreter.uri.fsPath,
+            ['-m', 'pip', 'install', '--upgrade', 'pip'],
+            { throwOnStdErr: false }
+        );
 
-            // Install deepnote-toolkit, ipykernel, and python-lsp-server in venv
-            logger.info(
-                `Installing deepnote-toolkit (${DEEPNOTE_TOOLKIT_VERSION}), ipykernel, and python-lsp-server in venv from PyPI`
-            );
-            this.outputChannel.appendLine(l10n.t('Installing deepnote-toolkit, ipykernel, and python-lsp-server...'));
+        if (pipUpgradeResult.stdout) {
+            logger.info(`pip upgrade output: ${pipUpgradeResult.stdout}`);
+        }
+        if (pipUpgradeResult.stderr) {
+            logger.info('pip upgrade stderr', pipUpgradeResult.stderr);
+        }
 
-            const installResult = await venvProcessService.exec(
-                venvInterpreter.uri.fsPath,
-                [
-                    '-m',
-                    'pip',
-                    'install',
-                    '--upgrade',
-                    `deepnote-toolkit[server]==${DEEPNOTE_TOOLKIT_VERSION}`,
-                    'ipykernel',
-                    'python-lsp-server[all]'
-                ],
-                { throwOnStdErr: false }
-            );
+        Cancellation.throwIfCanceled(token);
 
-            Cancellation.throwIfCanceled(token);
+        // Install deepnote-toolkit, ipykernel, and python-lsp-server in venv
+        logger.info(
+            `Installing deepnote-toolkit (${DEEPNOTE_TOOLKIT_VERSION}), ipykernel, and python-lsp-server in venv from PyPI`
+        );
+        this.outputChannel.appendLine(l10n.t('Installing deepnote-toolkit, ipykernel, and python-lsp-server...'));
 
-            if (installResult.stdout) {
-                this.outputChannel.appendLine(installResult.stdout);
-            }
-            if (installResult.stderr) {
-                this.outputChannel.appendLine(installResult.stderr);
-            }
+        const installResult = await venvProcessService.exec(
+            venvInterpreter.uri.fsPath,
+            [
+                '-m',
+                'pip',
+                'install',
+                '--upgrade',
+                `deepnote-toolkit[server]==${DEEPNOTE_TOOLKIT_VERSION}`,
+                'ipykernel',
+                'python-lsp-server[all]'
+            ],
+            { throwOnStdErr: false }
+        );
 
-            // Verify installation
-            const installedToolkitVersion = await this.isToolkitInstalled(venvInterpreter);
-            if (installedToolkitVersion != null) {
-                logger.info('deepnote-toolkit installed successfully in venv');
+        Cancellation.throwIfCanceled(token);
 
-                // Install kernel spec so the kernel uses this venv's Python
-                try {
-                    Cancellation.throwIfCanceled(token);
-                    await this.installKernelSpec(venvInterpreter, venvPath, token);
-                } catch (ex) {
-                    logger.warn('Failed to install kernel spec', ex);
-                    // Don't fail the entire installation if kernel spec creation fails
-                }
+        if (installResult.stdout) {
+            this.outputChannel.appendLine(installResult.stdout);
+        }
+        if (installResult.stderr) {
+            this.outputChannel.appendLine(installResult.stderr);
+        }
 
-                this.outputChannel.appendLine(l10n.t('✓ Deepnote toolkit ready'));
-                return { pythonInterpreter: venvInterpreter, toolkitVersion: installedToolkitVersion };
-            } else {
-                logger.error('deepnote-toolkit installation failed');
-                this.outputChannel.appendLine(l10n.t('✗ deepnote-toolkit installation failed'));
+        // Verify installation
+        const installedToolkitVersion = await this.isToolkitInstalled(venvInterpreter);
+        if (installedToolkitVersion != null) {
+            logger.info('deepnote-toolkit installed successfully in venv');
 
-                throw new DeepnoteToolkitInstallError(
-                    venvInterpreter.uri.fsPath,
-                    venvPath.fsPath,
-                    DEEPNOTE_TOOLKIT_VERSION,
-                    installResult.stdout || '',
-                    installResult.stderr || 'Package installation completed but verification failed'
-                );
-            }
-        } catch (ex) {
-            // If this is already a DeepnoteKernelError, rethrow it without wrapping
-            if (ex instanceof DeepnoteVenvCreationError || ex instanceof DeepnoteToolkitInstallError) {
-                throw ex;
+            // Install kernel spec so the kernel uses this venv's Python
+            try {
+                Cancellation.throwIfCanceled(token);
+                await this.installKernelSpec(venvInterpreter, venvPath, token);
+            } catch (ex) {
+                logger.warn('Failed to install kernel spec', ex);
+                // Don't fail the entire installation if kernel spec creation fails
             }
 
-            // Otherwise, log full details and wrap in a generic toolkit install error
-            logger.error('Failed to set up deepnote-toolkit', ex);
-            this.outputChannel.appendLine(l10n.t('Failed to set up deepnote-toolkit; see logs for details'));
+            this.outputChannel.appendLine(l10n.t('✓ Deepnote toolkit ready'));
+            return { pythonInterpreter: venvInterpreter, toolkitVersion: installedToolkitVersion };
+        } else {
+            logger.error('deepnote-toolkit installation failed');
+            this.outputChannel.appendLine(l10n.t('✗ deepnote-toolkit installation failed'));
 
             throw new DeepnoteToolkitInstallError(
-                baseInterpreter.uri.fsPath,
+                venvInterpreter.uri.fsPath,
                 venvPath.fsPath,
                 DEEPNOTE_TOOLKIT_VERSION,
-                '',
-                ex instanceof Error ? ex.message : String(ex),
-                ex instanceof Error ? ex : undefined
+                installResult.stdout || '',
+                installResult.stderr || 'Package installation completed but verification failed'
             );
         }
     }
@@ -365,7 +427,8 @@ export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller {
                 'import deepnote_toolkit; print(deepnote_toolkit.__version__)'
             ]);
             logger.info(`isToolkitInstalled result: ${result.stdout}`);
-            return result.stdout.trim();
+            const version = result.stdout.trim();
+            return version.length > 0 ? version : undefined;
         } catch (ex) {
             logger.debug('deepnote-toolkit not found', ex);
             return undefined;

src/kernels/deepnote/environments/deepnoteEnvironment.ts

@@ -28,6 +28,13 @@ export interface DeepnoteEnvironment {
      */
     venvPath: Uri;
 
+    /**
+     * Whether the venv is managed by this extension (created by us).
+     * If true, the venv can be deleted when the environment is removed.
+     * If false, the venv is external and should be preserved.
+     */
+    managedVenv: boolean;
+
     /**
      * Server information (set when server is running)
      */
@@ -71,6 +78,7 @@ export interface DeepnoteEnvironmentState {
         uri: string;
     };
     venvPath: string;
+    managedVenv?: boolean;
     createdAt: string;
     lastUsedAt: string;
     packages?: string[];