Further fixes to job submission

docs/config.yaml.template

@@ -138,6 +138,12 @@ session_cookie_secure: true
 #
 cluster:
   executor: local            # "local" or "lsf"
+#   extra_paths:               # Directories prepended to the server's PATH at
+#     - /opt/lsf/bin           # startup so scheduler commands (bsub, bjobs,
+#                              # bkill) are findable. Useful when running as a
+#                              # systemd service with a minimal default PATH.
+#   extra_env:                 # Extra environment variables set at startup.
+#     LSF_ENVDIR: /misc/lsf/conf  # Required for LSF to find lsf.conf.
 #   job_name_prefix: fg        # Prefix for cluster job names. REQUIRED for job
 #                              # reconnection after server restarts. Without this,
 #                              # active jobs will not be re-tracked and will

fileglancer/apps/core.py

@@ -6,6 +6,7 @@
 import shlex
 import shutil
 import subprocess
+from contextlib import nullcontext
 from pathlib import Path
 from datetime import datetime, UTC
 from typing import Optional
@@ -584,6 +585,8 @@
         # extra_args are handled via ResourceSpec in _build_resource_spec
         # to avoid double-application (config + per-job merge in py-cluster-api)
         config.pop("extra_args", None)
+        config.pop("extra_paths", None)
+        config.pop("extra_env", None)
         _executor = create_executor(**config)
     return _executor
 
@@ -788,12 +791,18 @@
 
 
 def _build_work_dir(job_id: int, app_name: str, entry_point_id: str,
-                    job_name_prefix: Optional[str] = None) -> Path:
-    """Build a working directory path under ~/.fileglancer/jobs/."""
+                    job_name_prefix: Optional[str] = None,
+                    username: Optional[str] = None) -> Path:
+    """Build a working directory path under ~/.fileglancer/jobs/.
+
+    When username is provided, expands ~username to the user's home directory
+    instead of the server process's home (which is typically root).
+    """
     safe_app = _sanitize_for_path(app_name)
     safe_ep = _sanitize_for_path(entry_point_id)
     prefix = f"{_sanitize_for_path(job_name_prefix)}-" if job_name_prefix else ""
-    return Path(os.path.expanduser(f"~/.fileglancer/jobs/{prefix}{job_id}-{safe_app}-{safe_ep}"))
+    home = os.path.expanduser(f"~{username}") if username else os.path.expanduser("~")
+    return Path(f"{home}/.fileglancer/jobs/{prefix}{job_id}-{safe_app}-{safe_ep}")
 
 
 async def submit_job(
@@ -810,6 +819,7 @@
     post_run: Optional[str] = None,
     container: Optional[str] = None,
     container_args: Optional[str] = None,
+    user_context=None,
 ) -> db.JobDB:
     """Submit a new job to the cluster.
 
@@ -891,25 +901,19 @@
 
         # Compute and persist work_dir now that we have the job ID
         work_dir = _build_work_dir(job_id, manifest.name, entry_point.id,
-                                   job_name_prefix=settings.cluster.job_name_prefix)
+                                   job_name_prefix=settings.cluster.job_name_prefix,
+                                   username=username)
         db_job.work_dir = str(work_dir)
         session.commit()
 
-    # Create work directory on disk
-    work_dir.mkdir(parents=True, exist_ok=True)
-
-    # Determine which repo to symlink and where to cd
+    # Pre-fetch repo into the shared server-owned cache. This must run as the
+    # server/root user because the cache lives under the server's home directory
+    # (~/.fileglancer/apps), which the authenticated user cannot write to.
     if manifest.repo_url:
-        # Tool code lives in a separate repo — clone it and cd to its root
-        tool_repo_dir = await _ensure_repo_cache(manifest.repo_url, pull=pull_latest)
-        repo_link = work_dir / "repo"
-        repo_link.symlink_to(tool_repo_dir)
+        cached_repo_dir = await _ensure_repo_cache(manifest.repo_url, pull=pull_latest)
         cd_suffix = "repo"
     else:
-        # Tool code is in the discovery repo — cd into manifest's subdirectory
-        repo_dir = await _ensure_repo_cache(app_url, pull=pull_latest)
-        repo_link = work_dir / "repo"
-        repo_link.symlink_to(repo_dir)
+        cached_repo_dir = await _ensure_repo_cache(app_url, pull=pull_latest)
         cd_suffix = f"repo/{manifest_path}" if manifest_path else "repo"
 
     # Build environment variable export lines
@@ -984,14 +988,21 @@
     resource_spec.stdout_path = str(work_dir / "stdout.log")
     resource_spec.stderr_path = str(work_dir / "stderr.log")
 
-    # Submit to executor
+    # Create work directory, symlink the cached repo, and submit to the cluster —
+    # all as the authenticated user so the job runs with correct ownership.
     executor = await get_executor()
     job_name = f"{manifest.name}-{entry_point.id}"
-    cluster_job = await executor.submit(
-        command=full_command,
-        name=job_name,
-        resources=resource_spec,
-    )
+    with user_context if user_context is not None else nullcontext():
+        work_dir.mkdir(parents=True, exist_ok=True)
+        repo_link = work_dir / "repo"
+        if repo_link.is_symlink() or repo_link.exists():
+            repo_link.unlink()
+        repo_link.symlink_to(cached_repo_dir)
+        cluster_job = await executor.submit(
+            command=full_command,
+            name=job_name,
+            resources=resource_spec,
+        )
 
     # Register callback to update DB when job reaches terminal state
     cluster_job.on_exit(_on_job_exit)

fileglancer/server.py

@@ -266,6 +266,22 @@ def mask_password(url: str) -> str:
         logger.debug(f"  external_proxy_url: {settings.external_proxy_url}")
         logger.debug(f"  atlassian_url: {settings.atlassian_url}")
 
+        # Prepend cluster extra_paths to PATH so scheduler commands
+        # (bsub, bjobs, bkill) are findable without relying on the
+        # system service's default PATH.
+        if settings.cluster.extra_paths:
+            extra = os.pathsep.join(settings.cluster.extra_paths)
+            os.environ["PATH"] = extra + os.pathsep + os.environ.get("PATH", "")
+            logger.debug(f"  cluster.extra_paths prepended to PATH: {extra}")
+
+        # Set extra environment variables needed by scheduler commands
+        # (e.g., LSF_ENVDIR for bsub to find lsf.conf). Pixi strips
+        # inherited env vars, so they must be set inside the process.
+        if settings.cluster.extra_env:
+            for key, value in settings.cluster.extra_env.items():
+                os.environ[key] = value
+                logger.debug(f"  cluster.extra_env set: {key}={value}")
+
         # Initialize database (run migrations once at startup)
         db.initialize_database(settings.db_url)
 
@@ -1687,22 +1703,22 @@ async def submit_job(body: JobSubmitRequest,
             if body.resources:
                 resources_dict = body.resources.model_dump(exclude_none=True)
 
-            with _get_user_context(username):
-                db_job = await apps_module.submit_job(
-                    username=username,
-                    app_url=body.app_url,
-                    entry_point_id=body.entry_point_id,
-                    parameters=body.parameters,
-                    resources=resources_dict,
-                    extra_args=body.extra_args,
-                    pull_latest=body.pull_latest,
-                    manifest_path=body.manifest_path,
-                    env=body.env,
-                    pre_run=body.pre_run,
-                    post_run=body.post_run,
-                    container=body.container,
-                    container_args=body.container_args,
-                )
+            db_job = await apps_module.submit_job(
+                username=username,
+                app_url=body.app_url,
+                entry_point_id=body.entry_point_id,
+                parameters=body.parameters,
+                resources=resources_dict,
+                extra_args=body.extra_args,
+                pull_latest=body.pull_latest,
+                manifest_path=body.manifest_path,
+                env=body.env,
+                pre_run=body.pre_run,
+                post_run=body.post_run,
+                container=body.container,
+                container_args=body.container_args,
+                user_context=_get_user_context(username),
+            )
             return _convert_job(db_job)
         except ValueError as e:
             raise HTTPException(status_code=400, detail=str(e))

fileglancer/settings.py

@@ -1,4 +1,4 @@
-from typing import List, Optional
+from typing import Dict, List, Optional
 from functools import cache
 import sys
 
@@ -14,6 +14,8 @@
 class ClusterSettings(BaseModel):
     """Cluster configuration matching py-cluster-api's ClusterConfig."""
     executor: str = 'local'
+    extra_paths: List[str] = []
+    extra_env: Dict[str, str] = {}
     cpus: Optional[int] = None
     gpus: Optional[int] = None
     memory: Optional[str] = None

frontend/package.json

@@ -1,7 +1,7 @@
 {
     "name": "fileglancer",
     "type": "module",
-    "version": "2.7.0-a4",
+    "version": "2.7.0-a9",
     "description": "Browse, share, and publish files on the Janelia file system",
     "keywords": [
         "ngff",