From bf21a520e6d584a86fc56d06e7a083b03e74bd8b Mon Sep 17 00:00:00 2001 From: "Eshan I." <2027eiyer@tjhsst.edu> Date: Fri, 17 Jul 2026 21:04:33 -0400 Subject: [PATCH 1/6] Add precision plumbing + /api/suggest-model (local-LLM model picker with heuristic fallback) --- flask-server/api/api_blueprint.py | 111 +++++++++++++++++++++++++++++- 1 file changed, 109 insertions(+), 2 deletions(-) diff --git a/flask-server/api/api_blueprint.py b/flask-server/api/api_blueprint.py index d74fd89..0446860 100644 --- a/flask-server/api/api_blueprint.py +++ b/flask-server/api/api_blueprint.py @@ -909,7 +909,7 @@ def _get_inference_job(session_id): return None -def _start_auto_segmentation(session_id, model_name, ct_file=None, server_input_path=None): +def _start_auto_segmentation(session_id, model_name, ct_file=None, server_input_path=None, precision="fp16"): if not _is_safe_id(session_id): return jsonify({"error": "Invalid session ID"}), 400 session_path = os.path.join(SESSIONS_DIR, session_id) @@ -960,7 +960,7 @@ def _on_gpu_slot(): try: output_mask_dir = run_auto_segmentation( input_path, session_dir=session_path, model=model_name, - session_id=session_id, on_start=_on_gpu_slot, + session_id=session_id, on_start=_on_gpu_slot, precision=precision, ) if _job_status() == "cancelled": @@ -1049,6 +1049,11 @@ def _pick_text(*keys): session_id = _pick_text("session_id", "SESSION_ID", "sessionId") or str(uuid.uuid4()) model_name = _pick_text("model_name", "model", "MODEL_NAME") or "ePAI" + # Weight precision for the GPU inference stage. "fp16" routes to the quantized + # half-precision weights; "fp32" keeps the original full-precision checkpoints. + precision = (_pick_text("precision", "PRECISION") or "fp16").lower() + if precision not in ("fp16", "fp32"): + precision = "fp16" uploaded_filename = _pick_text("uploaded_filename", "output_filename", "filename") input_server_path = _pick_text("INPUT_SERVER_PATH", "input_server_path", "server_path", "path") source_reconstruction_session_id = _pick_text("source_reconstruction_session_id") @@ -1109,9 +1114,111 @@ def _pick_text(*keys): model_name=model_name, ct_file=ct_file, server_input_path=input_server_path, + precision=precision, ) +# Catalog used by the automatic model picker. Each entry pairs a model with the +# tasks it is best suited for and its recommended inference precision. All of the +# segmentation networks run in FP16 by default (validated to produce masks +# identical to FP32); only the CPU-only ShapeKit stage is excluded. +MODEL_CATALOG = [ + {"model": "ePAI", "precision": "fp16", + "best_for": "pancreas, pancreatic duct, and pancreatic lesions (PDAC, cysts, PNETs)"}, + {"model": "SuPreM", "precision": "fp16", + "best_for": "broad multi-organ abdominal segmentation (25 structures)"}, + {"model": "MedFormer", "precision": "fp16", + "best_for": "26 abdominal structures plus pancreatic lesions; long-range context"}, + {"model": "R-Super", "precision": "fp16", + "best_for": "report-supervised segmentation of underrepresented structures"}, + {"model": "Atlas-Net", "precision": "fp16", + "best_for": "robust segmentation across varied CT protocols using anatomical priors"}, +] + +_VALID_MODELS = {m["model"] for m in MODEL_CATALOG} + + +def _heuristic_model_pick(hint: str): + """Rule-based fallback when the local LLM is unavailable.""" + text = (hint or "").lower() + if any(k in text for k in ("pancrea", "pdac", "cyst", "pnet", "duct", "lesion")): + choice = "ePAI" + elif any(k in text for k in ("organ", "abdomen", "abdominal", "liver", "kidney", + "spleen", "stomach", "multi", "whole", "full")): + choice = "SuPreM" + elif any(k in text for k in ("protocol", "robust", "varied", "generali")): + choice = "Atlas-Net" + else: + choice = "ePAI" + return { + "model": choice, + "precision": "fp16", + "reason": f"Rule-based match on your description; {choice} is best for " + + next(m["best_for"] for m in MODEL_CATALOG if m["model"] == choice) + + ". Running in FP16 (quantized) for speed.", + "source": "heuristic", + } + + +@api_blueprint.route('/suggest-model', methods=['POST']) +def suggest_model(): + """ + Automatically choose the best segmentation model (and precision) for a scan. + + Uses the local Ollama LLM to reason over the model catalog given a short, + optional free-text hint (e.g. "looking for pancreatic tumors") and/or the + uploaded filename. Falls back to a deterministic heuristic when the LLM is + unavailable, so the picker always returns a usable answer. + """ + payload = request.get_json(silent=True) or {} + hint = str(payload.get("hint") or request.form.get("hint") or "").strip() + filename = str(payload.get("filename") or request.form.get("filename") or "").strip() + combined_hint = " ".join(x for x in (hint, filename) if x) + + catalog_lines = "\n".join( + f"- {m['model']}: best for {m['best_for']}" for m in MODEL_CATALOG + ) + system_prompt = ( + "You are a routing assistant for a medical CT segmentation platform. " + "Choose exactly one model from the catalog that best fits the user's need. " + "Every model runs in FP16 (quantized) by default, which is faster and uses " + "half the memory with masks identical to FP32; only choose fp32 if the user " + "explicitly asks for maximum precision. Respond with a single JSON object: " + '{"model": , "precision": "fp16" or "fp32", ' + '"reason": }.' + ) + user_prompt = ( + f"Catalog:\n{catalog_lines}\n\n" + f"User need: {combined_hint or 'No specific description given; pick a sensible default.'}" + ) + + try: + result = chat_json( + model=DEFAULT_OLLAMA_MODEL, + system_prompt=system_prompt, + user_prompt=user_prompt, + temperature=0.1, + ) + model_choice = str(result.get("model", "")).strip() + if model_choice not in _VALID_MODELS: + return jsonify(_heuristic_model_pick(combined_hint)), 200 + precision = str(result.get("precision", "fp16")).strip().lower() + if precision not in ("fp16", "fp32"): + precision = "fp16" + reason = str(result.get("reason", "")).strip() or ( + f"{model_choice} is well suited to this task." + ) + return jsonify({ + "model": model_choice, + "precision": precision, + "reason": reason, + "source": "llm", + }), 200 + except (OllamaUnavailable, ValueError, Exception): + # Any LLM problem degrades gracefully to the deterministic heuristic. + return jsonify(_heuristic_model_pick(combined_hint)), 200 + + @api_blueprint.route('/inference-status/', methods=['GET']) def get_inference_status(session_id): job = _get_inference_job(session_id) From c3808b64fd210dccea5cf4eb0f454b86acfd8574 Mon Sep 17 00:00:00 2001 From: "Eshan I." <2027eiyer@tjhsst.edu> Date: Fri, 17 Jul 2026 21:05:48 -0400 Subject: [PATCH 2/6] Export BODYMAPS_PRECISION to GPU inference stages; ShapeKit (CPU) pinned to fp32 --- flask-server/services/auto_segmentor.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flask-server/services/auto_segmentor.py b/flask-server/services/auto_segmentor.py index 6a71ce0..d389d98 100644 --- a/flask-server/services/auto_segmentor.py +++ b/flask-server/services/auto_segmentor.py @@ -133,7 +133,7 @@ def _resolve_conda_activate_path(): return "" -def run_auto_segmentation(input_path, session_dir, model, session_id=None, on_start=None): +def run_auto_segmentation(input_path, session_dir, model, session_id=None, on_start=None, precision="fp16"): """ Dispatch to the appropriate model inference function. Serialized via _gpu_lock so concurrent requests queue instead of OOM-ing. @@ -154,6 +154,15 @@ def run_auto_segmentation(input_path, session_dir, model, session_id=None, on_st print(f"[on_start] callback error for {session_id}: {e}") if session_id: bind_session(session_id) + # Precision applies to the GPU inference stages only. The segmentation models + # (ePAI, SuPreM, MedFormer, R-Super, Atlas-Net) and the OpenVAE preprocessor are + # torch networks that can run in FP16; ShapeKit is CPU-only shape refinement, so + # precision is not meaningful there and is pinned to fp32. The value is exported so + # the inference subprocess (which inherits this environment) can load the matching + # half-precision weights. + effective_precision = "fp32" if model == "ShapeKit" else (precision or "fp16") + os.environ["BODYMAPS_PRECISION"] = effective_precision + print(f"[INFO] Inference precision for {model}: {effective_precision}") if model == 'ePAI': conda_path = _resolve_conda_activate_path() return _run_epai_inference( From a8c4a33d5636f8278eb9040c05448767145fdd92 Mon Sep 17 00:00:00 2001 From: "Eshan I." <2027eiyer@tjhsst.edu> Date: Fri, 17 Jul 2026 21:06:33 -0400 Subject: [PATCH 3/6] Upload UI: Auto (AI-picked) model default, plain-language precision toggle, hint field --- PanTS-Demo/src/routes/UploadPage.css | 69 ++++++++++++++++++++++++ PanTS-Demo/src/routes/UploadPage.tsx | 81 ++++++++++++++++++++++++++-- 2 files changed, 146 insertions(+), 4 deletions(-) diff --git a/PanTS-Demo/src/routes/UploadPage.css b/PanTS-Demo/src/routes/UploadPage.css index c200874..41e5888 100644 --- a/PanTS-Demo/src/routes/UploadPage.css +++ b/PanTS-Demo/src/routes/UploadPage.css @@ -1110,3 +1110,72 @@ border-top-color: #002D72; animation: upload-spin 0.75s linear infinite; } + +/* ── Simplified config: Auto hint, speed/quality toggle, result note ── */ +.simple-config { + display: flex; + flex-direction: column; + gap: 12px; + margin-top: 16px; +} + +.auto-hint-input { + width: 100%; + padding: 11px 14px; + border: 1px solid rgba(0, 45, 114, 0.16); + border-radius: 10px; + font-size: 13.5px; + font-family: inherit; + color: #1a2b47; + background: #fbfcfe; + outline: none; + transition: border-color 0.15s ease, box-shadow 0.15s ease; +} +.auto-hint-input:focus { + border-color: #002D72; + box-shadow: 0 0 0 3px rgba(0, 45, 114, 0.08); +} + +.precision-toggle { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 10px; +} +.precision-opt { + display: flex; + flex-direction: column; + align-items: flex-start; + gap: 3px; + padding: 12px 14px; + border: 1px solid rgba(0, 45, 114, 0.16); + border-radius: 10px; + background: #fff; + cursor: pointer; + text-align: left; + transition: border-color 0.15s ease, background 0.15s ease, box-shadow 0.15s ease; +} +.precision-opt:hover { border-color: rgba(0, 45, 114, 0.4); } +.precision-opt.active { + border-color: #002D72; + background: rgba(0, 45, 114, 0.05); + box-shadow: 0 0 0 2px rgba(0, 45, 114, 0.12); +} +.precision-opt-title { + font-size: 13.5px; + font-weight: 600; + color: #002D72; +} +.precision-opt-sub { + font-size: 11.5px; + color: #6b7793; +} + +.auto-note { + padding: 10px 13px; + border-radius: 10px; + background: rgba(45, 130, 90, 0.08); + border: 1px solid rgba(45, 130, 90, 0.2); + color: #1f5b3f; + font-size: 12.5px; + line-height: 1.45; +} diff --git a/PanTS-Demo/src/routes/UploadPage.tsx b/PanTS-Demo/src/routes/UploadPage.tsx index c009b9c..ca4714a 100644 --- a/PanTS-Demo/src/routes/UploadPage.tsx +++ b/PanTS-Demo/src/routes/UploadPage.tsx @@ -1,6 +1,7 @@ import React, { lazy, Suspense, useCallback, useEffect, useRef, useState } from 'react'; const MODEL_OPTIONS: { id: string; label: string; desc: string }[] = [ + { id: "Auto", label: "Auto — AI picks the best model", desc: "Recommended · a local AI reads your request and chooses for you" }, { id: "None", label: "None", desc: "View only — files never leave your browser" }, { id: "ePAI", label: "ePAI", desc: "For detailed pancreas and tumor analysis" }, { id: "SuPreM", label: "SuPreM", desc: "For whole-body scans from lungs to legs" }, @@ -79,7 +80,13 @@ const UploadPage: React.FC = () => { const [uploadProgress, setUploadProgress] = useState(0); const [isUploading, setIsUploading] = useState(false); const [inferenceCompleted, setInferenceCompleted] = useState(false); - const [selectedModel, setSelectedModel] = useState<"None" | "ePAI" | "SuPreM" | "OpenVAE" | "MedFormer" | "R-Super" | "Atlas-Net" | "">("None"); + const [selectedModel, setSelectedModel] = useState<"Auto" | "None" | "ePAI" | "SuPreM" | "OpenVAE" | "MedFormer" | "R-Super" | "Atlas-Net" | "">("Auto"); + // FP16 = quantized (faster, half the memory, identical masks); the default. + const [selectedPrecision, setSelectedPrecision] = useState<"fp16" | "fp32">("fp16"); + // Optional plain-language hint for the Auto picker, and the note it returns. + const [autoHint, setAutoHint] = useState(""); + const [autoNote, setAutoNote] = useState(""); + const [autoBusy, setAutoBusy] = useState(false); const [modelDropOpen, setModelDropOpen] = useState(false); const modelDropRef = useRef(null); const [preDropOpen, setPreDropOpen] = useState(false); @@ -412,6 +419,7 @@ const UploadPage: React.FC = () => { const inferFd = new FormData(); inferFd.append("session_id", sid); inferFd.append("model_name", model); + inferFd.append("precision", selectedPrecision); inferFd.append("uploaded_filename", uploadedName); const res = await fetch(`${API_BASE}/api/run-epai-inference`, { method: "POST", body: inferFd, signal: controller.signal, @@ -489,6 +497,7 @@ const UploadPage: React.FC = () => { const inferFd = new FormData(); inferFd.append("session_id", sid); inferFd.append("model_name", model); + inferFd.append("precision", selectedPrecision); inferFd.append("uploaded_filename", uploadedName); const res = await fetch(`${API_BASE}/api/run-epai-inference`, { method: "POST", body: inferFd, signal: controller.signal, @@ -541,7 +550,37 @@ const UploadPage: React.FC = () => { return; } - const model = selectedModel; + // Auto mode: ask the backend (local LLM, with a heuristic fallback) to choose + // the model + precision, then run with that choice. Resolves to a concrete + // model name before anything is uploaded. + let model: string = selectedModel; + let precision = selectedPrecision; + if (selectedModel === "Auto") { + try { + setAutoBusy(true); + setMessage("Auto-selecting the best model for your scan…"); + const filename = item ? (item.kind === 'dicom' ? item.label : item.file.name) : (path ? path.split("/").pop() : ""); + const res = await fetch(`${API_BASE}/api/suggest-model`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ hint: autoHint, filename }), + }); + const data = await parseApiResponse(res); + model = data.model || "ePAI"; + precision = (data.precision === "fp32" ? "fp32" : "fp16"); + setSelectedModel(model as typeof selectedModel); + setSelectedPrecision(precision); + setAutoNote(`Auto-selected ${model} · ${precision.toUpperCase()} — ${data.reason || "best fit for this scan."}`); + } catch { + model = "ePAI"; + precision = "fp16"; + setSelectedModel("ePAI"); + setAutoNote("Auto-select was unavailable, so it defaulted to ePAI · FP16."); + } finally { + setAutoBusy(false); + } + } + const sid = crypto.randomUUID(); const itemName = item ? (item.kind === 'dicom' ? item.label : item.file.name) : undefined; const label = bdmapId.trim() || (path ? path.split("/").pop() : itemName) || sid; @@ -567,6 +606,7 @@ const UploadPage: React.FC = () => { const fd = new FormData(); fd.append("session_id", sid); fd.append("model_name", model); + fd.append("precision", selectedPrecision); fd.append("INPUT_SERVER_PATH", path); const res = await fetch(`${API_BASE}/api/run-epai-inference`, { method: "POST", body: fd }); const data = await parseApiResponse(res); @@ -682,6 +722,7 @@ const UploadPage: React.FC = () => { const formData = new FormData(); formData.append("session_id", newSessionId); formData.append("model_name", "ePAI"); + formData.append("precision", selectedPrecision); formData.append("source_reconstruction_session_id", sessionId); try { @@ -969,12 +1010,44 @@ const UploadPage: React.FC = () => { + {/* ── Simple controls: Auto hint, speed/quality, result note ── */} +
+ {selectedModel === "Auto" && ( + setAutoHint(e.target.value)} + /> + )} +
+ + +
+ {autoNote &&
{autoNote}
} +
+ {/* ── Advanced options ── */}