From 330d89a4db40f4a5d0b1164467d0bc4e3562fd51 Mon Sep 17 00:00:00 2001 From: Aditya Sanjeev Date: Fri, 17 Jul 2026 23:23:40 -0700 Subject: [PATCH 1/2] Add CancerVerse dataset support to the viewer backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CancerVerse is a second dataset (22.8k CT-only cases, CV_%08d ids, CSV metadata) served alongside PanTS. Additive and dataset-dispatched — PanTS paths/behaviour are unchanged; CancerVerse is disabled unless CANCERVERSE_PATH is set. - constants.py: CANCERVERSE_PATH, CANCERVERSE_LOWRES_PATH, DATASET_PREFIXES. - utils.py: get_cancerverse_id / get_dataset_from_case_id / get_folder_id / get_case_nifti_paths; load CancerVerse CSV through the same _norm_cols into DF_CV (None if absent); select_dataset_df() switches search on ?dataset=. - api_blueprint.py: get-main-nifti serves CV CT + low-res from CANCERVERSE paths; mask endpoints (get-segmentations, get-specific-segmentations, get-report-data) return {"masks_available": false} for CV instead of crashing; previews degrade gracefully; /api/search honours ?dataset=. - services/npz_processor, preprocess_meshes: get_cancerverse_id parallels. - .env.example + README: document the new env vars. CancerVerse is CT-only for now, so any mask-dependent path is a graceful no-op. --- README.md | 13 +++- flask-server/.env.example | 28 ++++++++ flask-server/api/api_blueprint.py | 51 ++++++++++---- flask-server/api/utils.py | 77 +++++++++++++++++++++- flask-server/constants.py | 5 ++ flask-server/services/npz_processor.py | 11 +++- flask-server/services/preprocess_meshes.py | 11 +++- 7 files changed, 179 insertions(+), 17 deletions(-) create mode 100644 flask-server/.env.example diff --git a/README.md b/README.md index 33c570d..0057080 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ touch .env # creates the .env file nano .env ``` -Inside .env file: +Inside .env file (see `flask-server/.env.example` for the full list): ``` BASE_PATH=/ @@ -22,6 +22,17 @@ PANTS_PATH=/folder/where/PanTS USE_SSL=false ``` +Optional dataset vars: +``` +# Writable dir for precomputed PanTS low-res volumes (make_lowres.py output) +PANTS_LOWRES_PATH=/home/visitor/pants_lowres + +# CancerVerse (second, CT-only dataset). Leave unset to disable it. +CANCERVERSE_PATH=/folder/where/CancerVerse +CANCERVERSE_LOWRES_PATH=/home/visitor/cancerverse_lowres +``` +`CANCERVERSE_PATH` must contain `CancerVerse_dataset_metadata.csv` + `CV_########/ct.nii.gz`. When set, `/api/search?dataset=cancerverse` (or `dataset=all`) searches it; CancerVerse has no masks yet, so mask endpoints return `{"masks_available": false}`. + Run backend: ``` diff --git a/flask-server/.env.example b/flask-server/.env.example new file mode 100644 index 0000000..2fddae1 --- /dev/null +++ b/flask-server/.env.example @@ -0,0 +1,28 @@ +# Backend environment — copy to flask-server/.env and fill in. +# (.env is gitignored; never commit real values.) + +# --- Web app --- +BASE_PATH=/ +USE_SSL=false + +# --- PanTS dataset (primary) --- +# Folder that contains metadata.xlsx + image_only/ + mask_only/. +PANTS_PATH=/mnt/bodymaps/zzhou82/data/PanTS +# Writable dir for precomputed PanTS low-res volumes (make_lowres.py output). +PANTS_LOWRES_PATH=/home/visitor/pants_lowres + +# --- CancerVerse dataset (second, CT-only for now) --- +# Folder that contains CancerVerse_dataset_metadata.csv + CV_########/ct.nii.gz. +# Leave unset to disable CancerVerse (search/endpoints stay PanTS-only). +CANCERVERSE_PATH=/mnt/bodymaps/zzhou82/data/CancerVerse +# Writable dir for CancerVerse low-res volumes: +# /image_only/CV_########/ct_lowres.nii.gz +CANCERVERSE_LOWRES_PATH=/home/visitor/cancerverse_lowres + +# --- Database / sessions (fill in as needed) --- +# DB_USER= +# DB_PASS= +# DB_HOST= +# DB_NAME= +# SESSIONS_DIR_PATH=sessions +# PERMISSIONS_DIR=/home/visitor/data diff --git a/flask-server/api/api_blueprint.py b/flask-server/api/api_blueprint.py index d74fd89..5583141 100644 --- a/flask-server/api/api_blueprint.py +++ b/flask-server/api/api_blueprint.py @@ -200,7 +200,11 @@ def get_preview(clabel_ids): clabel_ids = clabel_ids.split(",") res = {} for clabel_id in clabel_ids: - pid = get_panTS_id(clabel_id) + try: + pid = get_folder_id(clabel_id) # dataset-aware (PanTS or CV) + except Exception: + res[clabel_id] = {"sex": "", "age": "", "tumor": 0} + continue entry = _METADATA_CACHE.get(pid, {"sex": "", "age": "", "tumor": 0}) res[clabel_id] = entry return jsonify(res) @@ -210,6 +214,9 @@ def get_preview(clabel_ids): def get_image_preview(clabel_id): if not _is_safe_id(clabel_id): return jsonify({"error": "Invalid id"}), 400 + if get_dataset_from_case_id(secure_filename(clabel_id)) == "CancerVerse": + # No profile thumbnails for CancerVerse yet — let the frontend fall back. + return jsonify({"error": "No preview for CancerVerse case"}), 404 path = os.path.join(Constants.PANTS_PATH, "profile_only", get_panTS_id(secure_filename(clabel_id)), "profile.jpg") if not os.path.exists(path): return jsonify({"error": f"File not found: {path} "}), 404 @@ -348,17 +355,27 @@ def get_mask_data(): def get_main_nifti(clabel_id): if not _is_safe_id(clabel_id): return jsonify({"error": "Invalid id"}), 400 - case_dir = f"{Constants.PANTS_PATH}/image_only/{get_panTS_id(secure_filename(clabel_id))}" - main_nifti_path = f"{case_dir}/{Constants.MAIN_NIFTI_FILENAME}" - # ?res=low → serve the precomputed low-res copy when present (much smaller/faster - # for big full-body scans). It lives under LOWRES_ROOT (a writable disk), NOT the - # read-only dataset mount. Falls back to full res if it hasn't been generated. - if (request.args.get('res') or '').strip().lower() == 'low': - low_name = Constants.MAIN_NIFTI_FILENAME.replace('.nii.gz', '_lowres.nii.gz') - low_path = f"{LOWRES_ROOT}/image_only/{get_panTS_id(secure_filename(clabel_id))}/{low_name}" - if os.path.exists(low_path): - main_nifti_path = low_path + safe_id = secure_filename(clabel_id) + want_low = (request.args.get('res') or '').strip().lower() == 'low' + + if get_dataset_from_case_id(safe_id) == "CancerVerse": + # CancerVerse: CT-only, stored flat at CANCERVERSE_PATH//ct.nii.gz. + paths = get_case_nifti_paths(safe_id) + main_nifti_path = paths["image"] + if want_low and os.path.exists(paths["lowres_image"]): + main_nifti_path = paths["lowres_image"] + else: + case_dir = f"{Constants.PANTS_PATH}/image_only/{get_panTS_id(safe_id)}" + main_nifti_path = f"{case_dir}/{Constants.MAIN_NIFTI_FILENAME}" + # ?res=low → serve the precomputed low-res copy when present (much smaller/faster + # for big full-body scans). It lives under LOWRES_ROOT (a writable disk), NOT the + # read-only dataset mount. Falls back to full res if it hasn't been generated. + if want_low: + low_name = Constants.MAIN_NIFTI_FILENAME.replace('.nii.gz', '_lowres.nii.gz') + low_path = f"{LOWRES_ROOT}/image_only/{get_panTS_id(safe_id)}/{low_name}" + if os.path.exists(low_path): + main_nifti_path = low_path if os.path.exists(main_nifti_path): response = make_response(send_file(main_nifti_path, mimetype='application/gzip')) @@ -464,6 +481,9 @@ def define_term(): @api_blueprint.route('/get-report-data/', methods=['GET']) def get_report_data(id): + # CancerVerse has no masks/RadGPT report yet — respond gracefully. + if get_dataset_from_case_id(secure_filename(str(id))) == "CancerVerse": + return jsonify({"masks_available": False}), 200 if id is None or not str(id).isdigit(): return jsonify({"error": "Invalid id parameter"}), 400 case_id = int(id) @@ -723,6 +743,8 @@ def get_report_data(id): @api_blueprint.route('/get-specific-segmentations/', methods=['POST']) async def get_specific_segmentations(combined_labels_id): + if get_dataset_from_case_id(combined_labels_id) == "CancerVerse": + return jsonify({"masks_available": False}), 200 combined_labels_id = combined_labels_id.replace("PanTS_", "") combined_labels_id = combined_labels_id.lstrip("0") try: @@ -753,6 +775,10 @@ async def get_specific_segmentations(combined_labels_id): async def get_segmentations(combined_labels_id): if not _is_safe_id(combined_labels_id): return jsonify({"error": "Invalid id"}), 400 + # CancerVerse has no masks yet — respond gracefully instead of 404/crashing so + # the viewer can show the CT alone. + if get_dataset_from_case_id(secure_filename(combined_labels_id)) == "CancerVerse": + return jsonify({"masks_available": False}), 200 nifti_path = f"{Constants.PANTS_PATH}/mask_only/{get_panTS_id(secure_filename(combined_labels_id))}/{Constants.COMBINED_LABELS_NIFTI_FILENAME}" labels = list(Constants.PREDEFINED_LABELS.values()) # ?res=low → serve the precomputed low-res mask (paired with the low-res CT so the @@ -1647,7 +1673,8 @@ def ping(): @api_blueprint.route("/search", methods=["GET"]) def api_search(): # return jsonify({"message": "pong"}), 200 - df = apply_filters(DF).copy() + # ?dataset=cancerverse|all switches/unions the base metadata; default = PanTS. + df = apply_filters(select_dataset_df()).copy() df = ensure_sort_cols(df) # ---- 排序參數 ---- diff --git a/flask-server/api/utils.py b/flask-server/api/utils.py index ace931a..096221a 100644 --- a/flask-server/api/utils.py +++ b/flask-server/api/utils.py @@ -53,9 +53,54 @@ def get_panTS_id(index): iter = max(0, 8 - len(index_str)) for _ in range(iter): cur_case_id = "0" + cur_case_id - cur_case_id = "PanTS_" + cur_case_id + cur_case_id = "PanTS_" + cur_case_id return cur_case_id +def get_cancerverse_id(index): + """CV_%08d id from a numeric index (accepts an optional 'CV_' prefix).""" + index_str = re.sub(r"^CV_?", "", str(index).strip(), flags=re.IGNORECASE) + if not re.fullmatch(r"\d+", index_str): + raise ValueError("Invalid case id: numeric value expected") + return "CV_" + index_str.zfill(8) + +def get_dataset_from_case_id(case_str): + """Dispatch on the id prefix: 'CancerVerse' for CV ids, else 'PanTS'.""" + return "CancerVerse" if str(case_str).strip().upper().startswith("CV") else "PanTS" + +def get_folder_id(case_id): + """Canonical on-disk folder id for either dataset from an incoming request id.""" + if get_dataset_from_case_id(case_id) == "CancerVerse": + return get_cancerverse_id(case_id) + return get_panTS_id(case_id) + +def get_case_nifti_paths(case_id): + """Resolve the CT / mask / low-res paths for a case, dispatching on dataset. + + CancerVerse is CT-only (no masks yet), so ``mask`` is None and + ``masks_available`` is False for CV cases. + """ + dataset = get_dataset_from_case_id(case_id) + if dataset == "CancerVerse": + folder = get_cancerverse_id(case_id) + return { + "dataset": dataset, + "folder_id": folder, + "image": f"{Constants.CANCERVERSE_PATH}/{folder}/{Constants.MAIN_NIFTI_FILENAME}", + "lowres_image": f"{Constants.CANCERVERSE_LOWRES_PATH}/image_only/{folder}/ct_lowres.nii.gz", + "mask": None, + "masks_available": False, + } + folder = get_panTS_id(case_id) + pants_lowres = os.environ.get("PANTS_LOWRES_PATH", "/home/visitor/pants_lowres") + return { + "dataset": dataset, + "folder_id": folder, + "image": f"{Constants.PANTS_PATH}/image_only/{folder}/{Constants.MAIN_NIFTI_FILENAME}", + "lowres_image": f"{pants_lowres}/image_only/{folder}/ct_lowres.nii.gz", + "mask": f"{Constants.PANTS_PATH}/mask_only/{folder}/{Constants.COMBINED_LABELS_NIFTI_FILENAME}", + "masks_available": True, + } + def clean_nan(obj): """Recursively replace NaN with None for JSON serialization.""" if isinstance(obj, dict): @@ -972,7 +1017,7 @@ def _norm_cols(df_raw: pd.DataFrame) -> pd.DataFrame: df = df_raw.copy() # ---- Case ID ---- - case_cols = ["PanTS ID", "PanTS_ID", "case_id", "id", "case", "CaseID"] + case_cols = ["PanTS ID", "PanTS_ID", "CancerVerse ID", "case_id", "id", "case", "CaseID"] def _first_nonempty(row, cols): for c in cols: if c in row.index and pd.notna(row[c]) and str(row[c]).strip(): @@ -1216,6 +1261,34 @@ def ensure_sort_cols(df: pd.DataFrame) -> pd.DataFrame: DF_RAW = pd.read_excel(META_FILE) DF = _norm_cols(DF_RAW) +# CancerVerse metadata (CT-only second dataset). Loaded through the SAME _norm_cols +# so search/sort/row_to_item work unchanged. Optional: if the path/CSV is absent +# (e.g. local dev) DF_CV stays None and CV search returns empty — never raises. +CANCERVERSE_META_FILE = ( + os.path.join(Constants.CANCERVERSE_PATH, "CancerVerse_dataset_metadata.csv") + if Constants.CANCERVERSE_PATH else None +) +DF_CV = None +if CANCERVERSE_META_FILE and os.path.exists(CANCERVERSE_META_FILE): + try: + DF_CV = _norm_cols(pd.read_csv(CANCERVERSE_META_FILE)) + except Exception as _cv_err: + print(f"[WARN] Could not load CancerVerse metadata: {_cv_err}") + DF_CV = None + +def select_dataset_df() -> pd.DataFrame: + """Pick the base DataFrame for search by the ?dataset= query param. + + 'pants' (default) → PanTS only (unchanged behaviour); 'cancerverse'/'cv' → CV; + 'all'/'both' → union of both. Falls back to PanTS when CV isn't loaded. + """ + ds = (request.args.get("dataset") or "").strip().lower() + if ds in ("cancerverse", "cv"): + return DF_CV if DF_CV is not None else DF.iloc[0:0] + if ds in ("all", "both"): + return pd.concat([DF, DF_CV], ignore_index=True) if DF_CV is not None else DF + return DF + def apply_filters(base: pd.DataFrame, exclude: Optional[Set[str]] = None) -> pd.DataFrame: exclude = exclude or set() df = base diff --git a/flask-server/constants.py b/flask-server/constants.py index b2dc8d2..88ef507 100644 --- a/flask-server/constants.py +++ b/flask-server/constants.py @@ -19,6 +19,11 @@ class Constants: # api_blueprint variables BASE_PATH = os.environ.get('BASE_PATH', '/') PANTS_PATH = os.environ.get('PANTS_PATH') + # Second dataset (CT-only, no masks yet). CV_%08d ids, CSV metadata. + # Low-res copies live under CANCERVERSE_LOWRES_PATH/image_only//ct_lowres.nii.gz. + CANCERVERSE_PATH = os.environ.get('CANCERVERSE_PATH') + CANCERVERSE_LOWRES_PATH = os.environ.get('CANCERVERSE_LOWRES_PATH', '/home/visitor/cancerverse_lowres') + DATASET_PREFIXES = {'PanTS': 'PanTS', 'CancerVerse': 'CV'} PERMISSIONS_DIR = os.environ.get('PERMISSIONS_DIR', "/home/visitor/data") MESH_PATH = PERMISSIONS_DIR + "/render_only" MAIN_NIFTI_FORM_NAME = 'MAIN_NIFTI' diff --git a/flask-server/services/npz_processor.py b/flask-server/services/npz_processor.py index a28f352..bfeed36 100644 --- a/flask-server/services/npz_processor.py +++ b/flask-server/services/npz_processor.py @@ -15,9 +15,18 @@ def get_panTS_id(index): iter = max(0, 8 - len(str(index))) for _ in range(iter): cur_case_id = "0" + cur_case_id - cur_case_id = "PanTS_" + cur_case_id + cur_case_id = "PanTS_" + cur_case_id return cur_case_id +def get_cancerverse_id(index): + """CV_%08d parallel to get_panTS_id (CancerVerse is CT-only — no masks yet).""" + digits = str(index).upper().replace("CV_", "").replace("CV", "") + return "CV_" + digits.zfill(8) + +def get_folder_id(index): + """Dataset-aware id: CV ids -> get_cancerverse_id, else get_panTS_id.""" + return get_cancerverse_id(index) if str(index).strip().upper().startswith("CV") else get_panTS_id(index) + def has_large_connected_component(slice_mask, threshold=8): """ Check if there is a connected component larger than a threshold in a 2D mask. diff --git a/flask-server/services/preprocess_meshes.py b/flask-server/services/preprocess_meshes.py index da1a54c..ba6f970 100644 --- a/flask-server/services/preprocess_meshes.py +++ b/flask-server/services/preprocess_meshes.py @@ -188,9 +188,18 @@ def get_panTS_id(index: int): iter = max(0, 8 - len(str(index))) for _ in range(iter): cur_case_id = "0" + cur_case_id - cur_case_id = "PanTS_" + cur_case_id + cur_case_id = "PanTS_" + cur_case_id return cur_case_id +def get_cancerverse_id(index): + """CV_%08d parallel to get_panTS_id (CancerVerse is CT-only — no masks yet).""" + digits = str(index).upper().replace("CV_", "").replace("CV", "") + return "CV_" + digits.zfill(8) + +def get_folder_id(index): + """Dataset-aware id: CV ids -> get_cancerverse_id, else get_panTS_id.""" + return get_cancerverse_id(index) if str(index).strip().upper().startswith("CV") else get_panTS_id(index) + # display_id: PanTS_00000900 def preprocess_case(display_id: str, label_nifti_path: str, output_root: str): label_nifti_path = Path(label_nifti_path) From 9856118a27572ee493d8540b44fcf746616ab961 Mon Sep 17 00:00:00 2001 From: Aditya Sanjeev Date: Fri, 17 Jul 2026 23:30:55 -0700 Subject: [PATCH 2/2] Fix CancerVerse metadata CSV path: it sits next to CANCERVERSE_PATH, not inside MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CancerVerse_dataset_metadata.csv lives in the parent of CANCERVERSE_PATH (/mnt/.../data/CancerVerse_dataset_metadata.csv), alongside the CancerVerse/ image folder — not within it. The old path never found it, so DF_CV stayed None and ?dataset=cancerverse always returned empty. Use dirname(normpath(...)) (also robust to a trailing slash) and correct the .env.example + README notes. --- README.md | 2 +- flask-server/.env.example | 3 ++- flask-server/api/utils.py | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0057080..2a9868d 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ PANTS_LOWRES_PATH=/home/visitor/pants_lowres CANCERVERSE_PATH=/folder/where/CancerVerse CANCERVERSE_LOWRES_PATH=/home/visitor/cancerverse_lowres ``` -`CANCERVERSE_PATH` must contain `CancerVerse_dataset_metadata.csv` + `CV_########/ct.nii.gz`. When set, `/api/search?dataset=cancerverse` (or `dataset=all`) searches it; CancerVerse has no masks yet, so mask endpoints return `{"masks_available": false}`. +`CANCERVERSE_PATH` holds the `CV_########/ct.nii.gz` cases; the metadata CSV `CancerVerse_dataset_metadata.csv` sits **next to** that folder (in its parent). When set, `/api/search?dataset=cancerverse` (or `dataset=all`) searches it; CancerVerse has no masks yet, so mask endpoints return `{"masks_available": false}`. Run backend: diff --git a/flask-server/.env.example b/flask-server/.env.example index 2fddae1..3757a55 100644 --- a/flask-server/.env.example +++ b/flask-server/.env.example @@ -12,7 +12,8 @@ PANTS_PATH=/mnt/bodymaps/zzhou82/data/PanTS PANTS_LOWRES_PATH=/home/visitor/pants_lowres # --- CancerVerse dataset (second, CT-only for now) --- -# Folder that contains CancerVerse_dataset_metadata.csv + CV_########/ct.nii.gz. +# Folder that contains the CV_########/ct.nii.gz cases. The metadata CSV lives +# NEXT TO this folder: /CancerVerse_dataset_metadata.csv. # Leave unset to disable CancerVerse (search/endpoints stay PanTS-only). CANCERVERSE_PATH=/mnt/bodymaps/zzhou82/data/CancerVerse # Writable dir for CancerVerse low-res volumes: diff --git a/flask-server/api/utils.py b/flask-server/api/utils.py index 096221a..586216a 100644 --- a/flask-server/api/utils.py +++ b/flask-server/api/utils.py @@ -1264,8 +1264,11 @@ def ensure_sort_cols(df: pd.DataFrame) -> pd.DataFrame: # CancerVerse metadata (CT-only second dataset). Loaded through the SAME _norm_cols # so search/sort/row_to_item work unchanged. Optional: if the path/CSV is absent # (e.g. local dev) DF_CV stays None and CV search returns empty — never raises. +# The CSV sits NEXT TO the CancerVerse image folder, not inside it: +# /CancerVerse_dataset_metadata.csv + /CancerVerse/CV_########/ CANCERVERSE_META_FILE = ( - os.path.join(Constants.CANCERVERSE_PATH, "CancerVerse_dataset_metadata.csv") + os.path.join(os.path.dirname(os.path.normpath(Constants.CANCERVERSE_PATH)), + "CancerVerse_dataset_metadata.csv") if Constants.CANCERVERSE_PATH else None ) DF_CV = None