diff --git a/Dockerfile b/Dockerfile index 877c1cc..4547406 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,6 +10,10 @@ RUN rm /opt/hippunfold_cache/trained_model.3d_fullres.Task102_hcp1200_T2w.nnUNet # Update OS and install prerequisite ENV DEBIAN_FRONTEND="noninteractive" +# Use archived repositories for Debian Buster (since it's EOL) +RUN sed -i 's|http://deb.debian.org/debian|http://archive.debian.org/debian|g' /etc/apt/sources.list && \ + sed -i 's|http://security.debian.org/debian-security|http://archive.debian.org/debian-security|g' /etc/apt/sources.list && \ + sed -i '/buster-updates/d' /etc/apt/sources.list RUN apt-get --allow-releaseinfo-change update RUN apt-get install -y python3-pip \ time \ @@ -38,6 +42,9 @@ RUN cd /app/ && conda run -n base /bin/bash -c "pip install -e ." # Set permissions for the entrypoint RUN chmod +x entrypoint.sh +ENV KEEP_DATA_PATH=1 +ENV SILENT=1 + ENTRYPOINT ["/bin/bash","entrypoint.sh"] diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..fe70743 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,131 @@ +# PolyForm Noncommercial License 1.0.0 + + + +## Acceptance + +In order to get any license under these terms, you must agree +to them as both strict obligations and conditions to all +your licenses. + +## Copyright License + +The licensor grants you a copyright license for the +software to do everything you might do with the software +that would otherwise infringe the licensor's copyright +in it for any permitted purpose. However, you may +only distribute the software according to [Distribution +License](#distribution-license) and make changes or new works +based on the software according to [Changes and New Works +License](#changes-and-new-works-license). + +## Distribution License + +The licensor grants you an additional copyright license +to distribute copies of the software. Your license +to distribute covers distributing the software with +changes and new works permitted by [Changes and New Works +License](#changes-and-new-works-license). + +## Notices + +You must ensure that anyone who gets a copy of any part of +the software from you also gets a copy of these terms or the +URL for them above, as well as copies of any plain-text lines +beginning with `Required Notice:` that the licensor provided +with the software. For example: + +> Required Notice: Copyright Yoyodyne, Inc. (http://example.com) + +## Changes and New Works License + +The licensor grants you an additional copyright license to +make changes and new works based on the software for any +permitted purpose. + +## Patent License + +The licensor grants you a patent license for the software that +covers patent claims the licensor can license, or becomes able +to license, that you would infringe by using the software. + +## Noncommercial Purposes + +Any noncommercial purpose is a permitted purpose. + +## Personal Uses + +Personal use for research, experiment, and testing for +the benefit of public knowledge, personal study, private +entertainment, hobby projects, amateur pursuits, or religious +observance, without any anticipated commercial application, +is use for a permitted purpose. + +## Noncommercial Organizations + +Use by any charitable organization, educational institution, +public research organization, public safety or health +organization, environmental protection organization, +or government institution is use for a permitted purpose +regardless of the source of funding or obligations resulting +from the funding. + +## Fair Use + +You may have "fair use" rights for the software under the +law. These terms do not limit them. + +## No Other Rights + +These terms do not allow you to sublicense or transfer any of +your licenses to anyone else, or prevent the licensor from +granting licenses to anyone else. These terms do not imply +any other licenses. + +## Patent Defense + +If you make any written claim that the software infringes or +contributes to infringement of any patent, your patent license +for the software granted under these terms ends immediately. If +your company makes such a claim, your patent license ends +immediately for work on behalf of your company. + +## Violations + +The first time you are notified in writing that you have +violated any of these terms, or done anything with the software +not covered by your licenses, your licenses can nonetheless +continue if you come into full compliance with these terms, +and take practical steps to correct past violations, within +32 days of receiving notice. Otherwise, all your licenses +end immediately. + +## No Liability + +***As far as the law allows, the software comes as is, without +any warranty or condition, and the licensor will not be liable +to you for any damages arising out of these terms or the use +or nature of the software, under any kind of legal claim.*** + +## Definitions + +The **licensor** is the individual or entity offering these +terms, and the **software** is the software the licensor makes +available under these terms. + +**You** refers to the individual or entity agreeing to these +terms. + +**Your company** is any legal entity, sole proprietorship, +or other kind of organization that you work for, plus all +organizations that have control over, are under the control of, +or are under common control with that organization. **Control** +means ownership of substantially all the assets of an entity, +or the power to direct its management and policies by vote, +contract, or otherwise. Control can be direct or indirect. + +**Your licenses** are all the licenses granted to you for the +software under these terms. + +**Use** means anything you do with the software requiring one +of your licenses. \ No newline at end of file diff --git a/README.md b/README.md index 9e15b49..9c11398 100644 --- a/README.md +++ b/README.md @@ -13,12 +13,8 @@ Note: - You will need the following demographic information (age at scan & sex) to run AID-HS on your patient's T1 MRI scan. - AID-HS has been developed on T1w scans acquired at 3T. It has not yet been thoroughly evaluated on 1.5T and 7T data -**SIGN UP TO THE AID-HS MAILING LIST**: -We request that all AID-HS users sign up to the mailing list. If you are using AID-HS, please send an email to `meld.study@gmail.com` with the subject 'Request to be added to the AID-HS mailing list' and provide use with your name and institute. This will ensure that we can update you about bug fixs and new releases. - -**EXISTING USERS: PLEASE UPDATE TO VERSION V1.0.1**: -We have released AID-HS V1.0.1 which fixes a couple of issues found by users. For more information about the release please see [AID-HS V1.0.1](https://github.com/MELDProject/AID-HS/releases/tag/v1.0.1). To update your code please follow the guidelines [Updating AID-HS to V1.0.1](https://aid-hs.readthedocs.io/en/latest/FAQs.html#Updating-AID-HS-to-V1.0.1) from our FAQ. - +**REGISTER TO GET YOUR AID-HS LICENSE**: +We request that all AID-HS users fill the [AID-HS registration form](https://docs.google.com/forms/d/e/1FAIpQLSdPbtraBZ2s0HD1W8qtF11wr_fYVTWZjraED03Rtl2ZjxeRMA/viewform?usp=header). Following registration you will received a license file. This file will be needed for use of all future AID-HS versions v1.1.0 and above. Your email address will be added to the AID-HS mailing list. This will ensure that we can update you about bugs fix and new releases. Pipeline overview:\ @@ -36,6 +32,7 @@ You can install and use the AID-HS pipeline with : - [**native installation**](https://aid-hs.readthedocs.io/en/latest/install_native.html): Not supported **YouTube tutorial available for the [docker and singularity installation](https://www.youtube.com/watch?v=RRAET7r05ys&t=11s&ab_channel=MELDproject)** +Note: for installation, please follow the online guidelines on github which are up to date compare to the videos **FAQs** If you have a question or if you are running into issues at any stage (installation/use/interpretation), have a look at our [FAQs](https://aid-hs.readthedocs.io/en/latest/FAQs.html) page as we may have already have a solution. @@ -56,7 +53,7 @@ Once installed you will be able to use the AID-HS pipeline on your data followin Features extracted from MRI scans from different MRI scanners have systematic differences between them. To remove scanner related biases we recommend harmonising your MRI data to the MRI data that was used in the AID-HS manuscript. This harmonisation is required for each MRI scanner / T1 sequence you are using. Notes: -- This step needs to be run only once, and requires data from at least 20 subjects acquired on the same scanner with the same T1 sequence and demographic information (e.g age and sex). See [harmonisation instructions](https://aid-hs.readthedocs.io/en/latest/harmonisation.html) for more details. +- This step needs to be run only once, and requires data from at least 20 controls acquired on the same scanner with the same T1 sequence and demographic information (e.g age and sex). See [harmonisation instructions](https://aid-hs.readthedocs.io/en/latest/harmonisation.html) for more details. - The AID-HS pipeline can also be run without harmonisation with no drop in performances. However, the characterisation of the hippocampal features compared to the normative growth curves will not be interpretable. @@ -69,7 +66,10 @@ An overview of the notebooks that we used to create the figures can be found [he ## Contacts -Mathilde Ripart, PhD \ +MELD project\ +`meld.study@gmail.com` + +Mathilde Ripart, PhD \ Research Fellow at UCL Great Ormond Street Institute of Child Health \ `m.ripart@ucl.ac.uk` @@ -80,3 +80,5 @@ Research Fellow at UCL Great Ormond Street Institute of Child Health \ + + diff --git a/aidhs/__init__.py b/aidhs/__init__.py index 719a859..9851c38 100644 --- a/aidhs/__init__.py +++ b/aidhs/__init__.py @@ -1,3 +1,3 @@ __author__ = __maintainer__ = "Mathilde Ripart" __email__ = "m.ripart@ucl.ac.uk" -__version__ = "1.0.1" \ No newline at end of file +__version__ = "1.1.0" \ No newline at end of file diff --git a/aidhs/aidhs_cohort_hip.py b/aidhs/aidhs_cohort_hip.py index 470fa13..0c61979 100755 --- a/aidhs/aidhs_cohort_hip.py +++ b/aidhs/aidhs_cohort_hip.py @@ -254,7 +254,7 @@ def get_sites(self): sites = [] for f in glob.glob(os.path.join(self.data_dir, "AIDHS_*")): if os.path.isdir(f): - sites.append(f.split("_")[-1]) + sites.append(f.split("AIDHS_")[-1]) return sites @contextmanager diff --git a/aidhs/data_preprocessing.py b/aidhs/data_preprocessing.py index fef5940..47926c1 100755 --- a/aidhs/data_preprocessing.py +++ b/aidhs/data_preprocessing.py @@ -201,7 +201,6 @@ def load_covars(self, subject_ids=None, demographic_file=DEMOGRAPHIC_FEATURES_FI covars = pd.DataFrame() ages = [] sex = [] - group = [] sites_scanners = [] for subject in subject_ids: subj = AidhsSubject(subject, cohort=self.cohort) @@ -215,12 +214,10 @@ def load_covars(self, subject_ids=None, demographic_file=DEMOGRAPHIC_FEATURES_FI sex.append(s) else: print(f'ERROR: There is an issue with the coded sex of subject {subject}') - group.append(subj.is_patient) sites_scanners.append(subj.site_code) # just site code now covars["ages"] = ages covars["sex"] = sex - covars["group"] = group covars["site_scanner"] = sites_scanners covars["ID"] = subject_ids @@ -497,7 +494,7 @@ def combat_whole_cohort(self, feature_name, outliers_file=None, combat_params_fi print(f"INFO: exclude subjects {np.array(self.subject_ids)[~combat_subject_include]}") if precombat_features: precombat_features = np.array(precombat_features) - # load in covariates - age, sex, group, site and scanner unless provided + # load in covariates - age, sex, site and scanner unless provided covars = self.covars[combat_subject_include].copy() # check for nan index_nan = pd.isnull(covars).any(1).to_numpy().nonzero()[0] @@ -518,7 +515,7 @@ def combat_whole_cohort(self, feature_name, outliers_file=None, combat_params_fi precombat_features.T, covars, batch_col="site_scanner", - categorical_cols=["sex", "group"], + categorical_cols=["sex"], continuous_cols="ages", ) # @@ -579,7 +576,7 @@ def get_combat_new_site_parameters(self,feature, demographic_file,): if len(np.array(listids)[np.array(combat_subject_include)])==0: print(f'Cannot compute harmonisation for {feature} because no subject found with this feature') return - # load in covariates - age, sex, group, site and scanner unless provided + # load in covariates - age, sex, site and scanner unless provided new_site_covars = self.load_covars(subject_ids=np.array(listids)[np.array(combat_subject_include)], demographic_file=demographic_file).copy() #check site_scanner codes are the same for all subjects if len(new_site_covars['site_scanner'].unique())==1: @@ -595,7 +592,7 @@ def get_combat_new_site_parameters(self,feature, demographic_file,): new_site_data = np.array(precombat_features).T dc.distributedCombat_site(new_site_data, bat, - new_site_covars[['ages','sex','group']], + new_site_covars[['ages','sex']], file=os.path.join(site_combat_path,f"{site_code}_{feature}_summary.pickle"), ref_batch = 'H0', robust=True,) @@ -606,12 +603,12 @@ def get_combat_new_site_parameters(self,feature, demographic_file,): ) # third, use variance estimates from full AIDHS cohort dc_out['var_pooled'] = pd.read_pickle(os.path.join(aidhs_combat_path,f'combat_{feature}_var.pickle')).ravel() - for c in ['ages','sex','group']: + for c in ['ages','sex']: new_site_covars[c]=new_site_covars[c].astype(np.float64) print('step3') pickle_file = os.path.join(site_combat_path,f"{site_code}_{feature}_harmonisation_params_test.pickle") _=dc.distributedCombat_site( - pd.DataFrame(new_site_data), bat, new_site_covars[['ages','sex','group']], + pd.DataFrame(new_site_data), bat, new_site_covars[['ages','sex']], file=pickle_file, central_out=dc_out, ref_batch = 'H0', diff --git a/aidhs/download_data.py b/aidhs/download_data.py new file mode 100644 index 0000000..b8deaf5 --- /dev/null +++ b/aidhs/download_data.py @@ -0,0 +1,40 @@ +import urllib.request +import os +import numpy as np +from aidhs.paths import DATA_PATH +import sys +import shutil +import tempfile + +def _fetch_url(url, fname): + def dlProgress(count, blockSize, totalSize): + percent = int(count*blockSize*100/totalSize) + if not "SILENT" in os.environ: + sys.stdout.write("\r" + url + "...%d%%" % percent) + sys.stdout.flush() + return urllib.request.urlretrieve(url, fname, reporthook=dlProgress) + + +def download_aidhs_data(aidhs_data_path=DATA_PATH): + """ + download AID-HS data from GitHub release: model, parameters and test data + """ + url = "https://github.com/MELDProject/AID-HS/releases/download/aidhs_data/aidhs_data.zip" + with tempfile.TemporaryDirectory() as tmpdirname: + # download to tmpdir + _fetch_url(url, os.path.join(tmpdirname, "aidhs_data_folder.zip")) + # unpack + shutil.unpack_archive(os.path.join(tmpdirname, "aidhs_data_folder.zip"), aidhs_data_path) + print(f"\ndownloaded AID-HS data to {aidhs_data_path}") + +def check_data(force_download=False): + for folder in ['input','output','models','params']: + exit = False + if os.path.exists(os.path.join(DATA_PATH, folder)): + print(f'The folder {folder} already exists at {DATA_PATH}.') + exit = True + if force_download: + print('Data to download already (partially) exists. \nData will be overwritten.') + if exit and (force_download==False): + print('Data to download already (partially) exists. \nDownload aborted. Please delete folders or provide a new path.') + sys.exit() \ No newline at end of file diff --git a/aidhs/paths.py b/aidhs/paths.py index 0dcf552..85b8b7b 100755 --- a/aidhs/paths.py +++ b/aidhs/paths.py @@ -57,7 +57,7 @@ # paths to important data files - relative to BASE_PATH -DEMOGRAPHIC_FEATURES_FILE = os.path.join(DATA_PATH, "demographics_file.csv") +DEMOGRAPHIC_FEATURES_FILE = f"/tmp/demographics_file_{os.getpid()}.csv" # params file CLIPPING_PARAMS_FILE='clipping_parameters_sigma.json' diff --git a/aidhs/test/test_aidhs_license.py b/aidhs/test/test_aidhs_license.py new file mode 100644 index 0000000..2f8da3f --- /dev/null +++ b/aidhs/test/test_aidhs_license.py @@ -0,0 +1,31 @@ +import os +import sys +import re +from pathlib import Path + +def test_license(): + + # Get the meld license variable + aidhs_license_file = os.getenv("AIDHS_LICENSE", None) + + if aidhs_license_file is None: + print('ERROR: Could not find a AIDHS_LICENSE environment variable. Please ensure you have exported the AIDHS_LICENSE environment following the AID-HS installation guidelines') + sys.exit() + if not os.path.isfile(aidhs_license_file): + print(f'ERROR: The file {aidhs_license_file} does not exist.\nPlease ensure you got the meld license file by filling the registration form provided in the AID-HS installation guidelines and provided the right path to the file') + sys.exit() + + # check that the license is correct + text = Path(aidhs_license_file).read_text() + m = re.search(r"License\s*ID[:\s]*([0-9]+)", text, re.IGNORECASE) + if m: + license_id = m.group(1) + if not len(license_id) == 6: + print("ERROR: The license ID provided does not seem correct.\nPlease ensure you got the correct meld license file by filling the registration form provided in the AID-HS installation guidelines and provided the right path to the file") + sys.exit() + else: + print(f"ERROR: The license file {aidhs_license_file} does not seem correct.\nPlease ensure you got the correct meld license file by filling the registration form provided in the AID-HS installation guidelines and provided the right path to the file") + sys.exit() + +# call the test +test_license() \ No newline at end of file diff --git a/compose.yml b/compose.yml index 5496e68..a1e4beb 100644 --- a/compose.yml +++ b/compose.yml @@ -1,9 +1,13 @@ services: aidhs: - image: meldproject/aidhs:latest + image: meldproject/aidhs:v1.1.0 platform: "linux/amd64" volumes: - volumes:/data + environment: + - AIDHS_LICENSE=/run/secrets/aidhs_license.txt + secrets: + - aidhs_license.txt user: $DOCKER_USER deploy: resources: @@ -12,3 +16,7 @@ services: - capabilities: [gpu] count: 0 +secrets: + aidhs_license.txt: + file: ./aidhs_license.txt + diff --git a/docs/FAQs.md b/docs/FAQs.md index 95be819..2a9f138 100644 --- a/docs/FAQs.md +++ b/docs/FAQs.md @@ -4,6 +4,40 @@ ## **Issues & questions with installation** +### **Issue with the AID-HS License** + +- If your issue is: + ```bash + ERROR: Could not find a AIDHS_LICENSE environment variable. Please ensure you have exported the AIDHS_LICENSE environment following the AID-HS installation guidelines + ``` + This means that the AIDHS_LICENSE variable has not been exported/initialised in the environment (terminal) you are using. The variable should be automatically exported when using compose.yml file for the Docker. For native installation, you can export manually the environment variable by doing: + ```bash + export AIDHS_LICENSE= + ``` + Please contact the team if the issue continues + +- If your issue is: + ```bash + ERROR: The file aidhs_license.txt does not exist. + Please ensure you got the AID-HS license file by filling the registration form provided in the AID-HS installation guidelines and provided the right path to the file + ``` + This means that the aidhs_license.txt cannot be found in the main aidhs folder (where the code is). This can happen if you did not get the AID-HS license file or if the file was placed in an incorect folder. To get the proper AID-HS license ID, please fill in to the [AID-HS registration form](https://docs.google.com/forms/d/e/1FAIpQLSdPbtraBZ2s0HD1W8qtF11wr_fYVTWZjraED03Rtl2ZjxeRMA/viewform?usp=header). + +- If your issue is: + ```bash + ERROR: The license ID provided does not seem correct. + Please ensure you got the correct AID-HS license file by filling the registration form provided in the AID-HS installation guidelines and provided the right path to the file + ``` + or + ```bash + ERROR: The license file aidhs_license.txt does not seem correct. + Please ensure you got the correct AID-HS license file by filling the registration form provided in the AID-HS installation guidelines and provided the right path to the file + + ``` + This means that the AID-HS license file exists and is at the correct place, but the license ID does not exist or is incorrect. To get the proper AID-HS license ID, please fill in the [AID-HS registration form](https://docs.google.com/forms/d/e/1FAIpQLSdPbtraBZ2s0HD1W8qtF11wr_fYVTWZjraED03Rtl2ZjxeRMA/viewform?usp=header). + + + ### **Issue with Singularity - Not enough space when with creating the SIF** ```bash INFO: Creating SIF file... @@ -58,23 +92,26 @@ docker-compose down --remove-orphans --- -## **Updating AID-HS to V1.0.1** +## **Updating AID-HS to V1.1.0** -The instructions below are for users that already have used AID-HS v1.0.0 on patients and would like to update to AID-HS v1.0.1 while keeping the same aidhs_data_folder folder. +The instructions below are for users that already have used AID-HS v1.0.1 on patients and would like to update to AID-HS v1.1.0 while keeping the same aidhs_data_folder folder. +### ** Register to get your AID-HS license**: +Register at [AID-HS registration form](https://docs.google.com/forms/d/e/1FAIpQLSdPbtraBZ2s0HD1W8qtF11wr_fYVTWZjraED03Rtl2ZjxeRMA/viewform?usp=header). Following registration you will received a license file. This file will be needed for use of all future AID-HS versions v1.1.0 and above. Your email address will be added to the AID-HS mailing list. This will ensure that we can update you about bugs fix and new releases. ### 📥 **Get the updated code** -Depending on wether you previously downloaded `V1.0.0` as a zip/tar folder or used Git to download the code, you will need to follow the same route to get the update `v1.0.1` code. +Please follow the Download method below to get the new code ::::{tab-set} :::{tab-item} Download -1. Go to the [github releases page](https://github.com/MELDProject/AID-HS/releases) and download the latest source zip or tar of version `V1.0.1`. -2. Extract the folder `AID-HS-1.0.1` -3. Copy the files below from your old `AID-HS-1.0.0` directory to your new `AID-HS-1.0.1` directory: +1. Go to the [github releases page](https://github.com/AID-HSProject/AID-HS/releases) and download the latest version `V1.1.0`, by clicking on `Source code (zip)` or `Source code (tar.gz)`. +2. Extract the folder `AID-HS-1.1.0` +3. Copy the files below from your old `AID-HS-1.0.1` directory to your new `AID-HS-1.1.0` directory: - the `compose.yml` - the `config.ini` +4. Copy the `aidhs_license.txt` into the extracted folder (see above how to get the AID-HS license) ::: :::{tab-item} Git @@ -85,10 +122,11 @@ git stash git pull git stash pop ``` +3) Copy the `aidhs_license.txt` into the extracted folder (see above how to get the AID-HS license) ::: :::: -Then depending on if you have a Native, Docker or Singularity installation of AID-HS `v1.0.0` you will need to follow the same type of installation to update to `v1.0.1`: +Then depending on if you have a Native, Docker or Singularity installation of AID-HS `v1.0.1` you will need to follow the same type of installation to update to `v1.1.0`: ::::{tab-set} @@ -100,7 +138,7 @@ Then depending on if you have a Native, Docker or Singularity installation of AI ``` conda activate aidhs ``` -2. Update the code package in the environment. Make sure you are in the new `AID-HS-1.0.1` directory and run: +2. Update the code package in the environment. Make sure you are in the new `AID-HS-1.1.0` directory and run: ``` pip install -e . ``` @@ -112,7 +150,7 @@ pip install -e . **🐳 Docker Users:** You will need to pull the latest docker image ```bash -docker pull meldproject/aidhs:latest +docker pull MELDproject/aidhs:latest ``` ::: @@ -122,7 +160,7 @@ docker pull meldproject/aidhs:latest **🚀 Singularity Users:** You will need to pull the latest image ```bash -singularity pull docker://meldproject/aidhs:latest +singularity pull docker://MELDproject/aidhs:latest ``` ::: :::: @@ -133,14 +171,14 @@ Follow the guidelines **"Verify installation"** to run the test again. - 🐳[Docker Users](https://aid-hs.readthedocs.io/en/latest/install_docker.html#verify-installation) - 🚀[Singularity Users](https://aid-hs.readthedocs.io/en/latest/install_singularity.html#verify-installation) -### 🧠 **Update your predictions with the registration fix** +### 🧠 **Update your predictions with the harmonisation parameters fix** If you want to update the predictions with the new registration for patients you have already ran through AID-HS, please follow the instructions bellow: -1) Create a list of ids of patients you want to rerun: e.g. `list_subjects_rerun_v1.0.1.txt` +1) Create a list of ids of patients you want to rerun: e.g. `list_subjects_rerun_v1.1.0.txt` -2) Then run one of the commands below. It will recreate the PDF report for your patient. +2) Then run the pipeline again. It will recreate the PDF report for your patient. -**WARNING** This will overwrite the files and the patient report in `output/predictions_reports` +**WARNING** This will overwrite the data for that patient and the patient report in `output/predictions_reports` ::::{tab-set} @@ -149,7 +187,7 @@ If you want to update the predictions with the new registration for patients you **💻 Native Installation Users:** ```bash -python scripts/new_patient_pipeline/run_pipeline_prediction.py -ids list_subjects_rerun_v1.0.1.txt +python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects_rerun_v1.1.0.txt -demos demographics_file.csv ``` ::: @@ -158,7 +196,7 @@ python scripts/new_patient_pipeline/run_pipeline_prediction.py -ids list_subject **🐳 Docker Users:** ```bash -DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/run_pipeline_prediction.py -ids list_subjects_rerun_v1.0.1.txt +DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects_rerun_v1.1.0.txt -demos demographics_file.csv ``` ::: @@ -167,7 +205,7 @@ DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_pati **🚀 Singularity Users:** ```bash -singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/run_pipeline_prediction.py -ids list_subjects_rerun_v1.0.1.txt" +singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects_rerun_v1.1.0.txt -demos demographics_file.csv" ``` ::: :::: diff --git a/docs/harmonisation.md b/docs/harmonisation.md index d07b226..bebfd07 100644 --- a/docs/harmonisation.md +++ b/docs/harmonisation.md @@ -9,10 +9,11 @@ Here is the video tutorial about how to run the harmonisation using the AID-HS p ## Compute the harmonisation paramaters -The harmonisation parameters are computed using [Distributed Combat](https://doi.org/10.1016/j.neuroimage.2021.118822). -To get these parameters you will need a cohort of subjects acquired from the same scanner and under the same protocol (sequence, parameters, ...). -Subjects can be controls and/or patients, but we advise to use ***at least 20 subjects*** to enable an accurate harmonisation, and to not use HS patients for the harmonisation. -Try to ensure the data are high quality (i.e no blurring, no artefacts, no cavities in the brain). +The harmonisation parameters are computed using [Distributed Combat](https://doi.org/10.1016/j.neuroimage.2021.118822).\ +To get these parameters you will need a cohort of subjects acquired from the same scanner and under the same protocol (sequence, parameters, ...).\ +Subjects can be **controls** and/or **patients with no hippocampal abnormalities** (e.g. frontal FCD). Do not use HS patients for the harmonisation.\ +We advise to use ***at least 20 subjects*** to enable an accurate harmonisation.\ +Try to ensure the data are high quality (i.e no blurring, no artefacts, no cavities in the brain).\ Demographic information (e.g age and sex) will be required for this process, follow the [guidelines](https://aid-hs.readthedocs.io/en/latest/prepare_data.html) WARNING: zero variance in the demographics information (e.g. having the same age for all subjects) will lead to Combat failures or errors. @@ -22,7 +23,7 @@ Once you have done the process once, you can follow the [general guidelines to p ## Running - Ensure you have installed the AID-HS pipeline with [docker container](https://aid-hs.readthedocs.io/en/latest/install_docker.html). -- **Chose a harmonisation** code for this scanner starting by 'H' (e.g H1, H2, ..). This harmonisation code will be needed to organise your data and run the code as detailled below. +- **Chose a harmonisation** code for this scanner starting by 'H' (e.g H1, H2, ..). Avoid underscores '_' in the code. This harmonisation code will be needed to organise your data and run the code as detailled below. - Ensure you have [organised your MRI data](https://aid-hs.readthedocs.io/en/latest/prepare_data.html#prepare-the-mri-data-in-bids-format-mandatory) and [provided demographic information](https://aid-hs.readthedocs.io/en/latest/prepare_data.html#prepare-the-demographic-information-to-run-the-harmonisation) before running this pipeline. diff --git a/docs/install_docker.md b/docs/install_docker.md index fc5a622..45ddb65 100644 --- a/docs/install_docker.md +++ b/docs/install_docker.md @@ -1,16 +1,13 @@ # Docker container - -**WARNING: Installation and use has not yet been tested on Windows. Please let us know if you have succeeded or are having challenges using the docker container on Windows** - The Docker container has all the prerequisites embedded into it. This makes it easier to install and compatible with most OS systems. Notes: -- Currently only tested on **Linux** (HPC Singularity coming soon) +- tested on **Linux**, **Windows** - You will need **~20GB of space** to install the container - The docker image contains Miniconda 3, HippUnfold v1.1.0, and AID-HS. The whole image is ~18GB. -Here is the video tutorial demonstrating how to do the docker installation - [Docker and Singularity Installation of AID-HS Tutorial](https://www.youtube.com/watch?v=RRAET7r05ys&t=11s&ab_channel=MELDproject). +Here is the video tutorial demonstrating how to do the docker installation - [Docker and Singularity Installation of AID-HS Tutorial](https://www.youtube.com/watch?v=RRAET7r05ys&t=11s&ab_channel=MELDproject). Note that some part of the installation video are outdated for AID-HS > v1.1.0. Please follow the github guidelines for up to date instructions. ## Prerequisites @@ -29,12 +26,15 @@ Enabling your computer's GPUs for running the pipeline accelerates the HippUnfol Install the [*nvidia container toolkit*](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html). +## AID-HS license +In order to run AID-HS you need to have a `aidhs_license.txt` in the aidhs folder. To get this file, please fill out the [AID-HS registration form](https://docs.google.com/forms/d/e/1FAIpQLSdPbtraBZ2s0HD1W8qtF11wr_fYVTWZjraED03Rtl2ZjxeRMA/viewform?usp=header). Once submitted, your application will be automatically reviewed and the aidhs_license.txt file will be send to your email. + ## Installation & configuration In order to run the docker, you'll need to configure a couple of files 1. Download `aidhs.zip` from the [latest github release](https://github.com/MELDProject/AID-HS/releases/latest) and extract it. -2. Download the aidhs_data_folder from [figshare](https://figshare.com/s/48c92b1b53f8f0c67dec) -3. Unzip the folder where you want to store the aidhs_data_folder +2. Copy the `aidhs_license.txt` into the extracted folder (see above how to get the AID-HS license) +3. Create the aidhs_data folder, if it doesn't exist already. This folder is where you would like to store MRI data to run the classifier 4. In the AID-HS folder, open and edit the compose.yml to add the path to the aidhs_data_folder. The initial compose.yml file looks like : ``` services: @@ -43,6 +43,10 @@ services: platform: "linux/amd64" volumes: - volumes:/data + environment: + - AIDHS_LICENSE=/run/secrets/aidhs_license.txt + secrets: + - aidhs_license.txt user: $DOCKER_USER deploy: resources: @@ -51,6 +55,10 @@ services: - capabilities: [gpu] count: 0 +secrets: + aidhs_license.txt: + file: ./aidhs_license.txt + ``` Change the line below "`volumes:`" to point to the aidhs_data_folder. Do not delete the "`:/data`" at the end.\ For example, if you wanted the folder to be on a mounted drive such as "`/mnt/datadrive/aidhs-data`" you should change the line as showed below: @@ -71,7 +79,7 @@ On windows, if you're using absolute paths, use forward slashes and quotes: ::: -5. **WARNING:** If you do not have GPU on your computer (e.g. Mac laptop) you will need to open the compose.yml file and remove the last 6th lines of the text (everything that includes `deploy` and below).\ +5. **WARNING:** If you do not have GPU on your computer (e.g. Mac laptop) you will need to open the compose.yml file and remove the 6th lines of the text below (everything that includes `deploy` up to secrets).\ Your file should look like that: ``` services: @@ -80,12 +88,48 @@ services: platform: "linux/amd64" volumes: - volumes:/data + environment: + - AIDHS_LICENSE=/run/secrets/aidhs_license.txt + secrets: + - aidhs_license.txt user: $DOCKER_USER + +secrets: + aidhs_license.txt: + file: ./aidhs_license.txt ``` 6. **WARNING** If you are running docker with Docker Desktop, you will need to ensure that the memory usage allowed by docker is to the maximum, as Docker Desktop halves the memory usage by default. For that you can go in the Docker Desktop settings and change the memory limit (more help in this [post](https://stackoverflow.com/questions/43460770/docker-windows-container-memory-limit)) -## Download AID-HS docker & Verify installation +## Set up paths and download model +Before being able to use AID-HS on your data, data paths need to be set up and the pretrained model needs to be downloaded. + +1. Make sure you have 20GB of storage space available for the docker, and 1GB available for the aidhs data. +2. Run this command to download the docker image and the data folder + +::::{tab-set} + +:::{tab-item} Linux +:sync: linux +```bash +DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/prepare_aidhs.py +``` +::: + +:::{tab-item} Windows +:sync: windows +```bash +docker compose run aidhs python scripts/new_patient_pipeline/prepare_aidhs.py +``` +::: + +:::: + +:::{note} +::: + +## Verify installation + The line below will download AID-HS and then run a test to verify that everything is installed and set up properly. It may take up to an 1h to download the docker image and then takes approximately 1 minute to run the test. ::::{tab-set} @@ -154,4 +198,4 @@ Please see our [FAQs](https://aid-hs.readthedocs.io/en/latest/FAQs.html) for com ## Contact -If you encounter any errors, please contact `m.ripart@ucl.ac.uk` for support +If you encounter any errors, please contact `meld.study@gmail.com` for support diff --git a/docs/install_native.md b/docs/install_native.md index 1a11575..85e6b54 100644 --- a/docs/install_native.md +++ b/docs/install_native.md @@ -17,20 +17,14 @@ AID-HS extracts volume- and surface-based features of the hippocampus using **Hi ### Workbench Connectom AID-HS uses **Workbench Connectom** to create additional surface-based features. Please follow instructions to [install Workbench Connectom](https://www.humanconnectome.org/software/get-connectome-workbench). +## AID-HS license +In order to run AID-HS you need to have a `aidhs_license.txt` in the aidhs folder. To get this file, please fill out the [AID-HS registration form](https://docs.google.com/forms/d/e/1FAIpQLSdPbtraBZ2s0HD1W8qtF11wr_fYVTWZjraED03Rtl2ZjxeRMA/viewform?usp=header). Once submitted, your application will be automatically reviewed and the aidhs_license.txt file will be send to your email. + ## Installation & configuration In order to run the pipeline, you'll need to configure a couple of files -1. Download `aidhs.zip` from the [latest github release](https://github.com/MELDProject/aidhs/releases/latest) and extract it. -2. Download the aidhs_data_folder from [figshare](https://figshare.com/s/48c92b1b53f8f0c67dec) -3. Unzip the folder where you want to store the aidhs_data_folder -4. Open the file`'config.ini` and replace the line: -``` -data_path = /data -``` -by -``` -data_path = -``` +1. Download `aidhs.zip` from the [latest github release](https://github.com/MELDProject/AID-HS/releases/latest) and extract it. +2. Copy the `aidhs_license.txt` into the extracted folder (see above how to get the AID-HS license) ### Create the environment @@ -47,6 +41,16 @@ conda activate aidhs pip install -e . ``` +## Set up paths and download model + +Before being able to use AID-HS on your data, some paths need to be set up and the pretrained model needs to be downloaded. For this, run: +```bash +python prepare_aidhs.py +``` + +This script will ask you if you want to change the path to the data folder, answer **'y'** for yes. \ +Then, it will ask for the the location of your **AID-HS data folder** where you would like to store MRI data to run the classifier. Create the **AID-HS data folder**, if it doesn't exist, and provide the path. It will download the pretrained model and test data to a folder inside your AID-HS data folder + ## Verify installation To verify that you have installed all packages, set up paths correctly, and downloaded all data, this verification script will run the pipeline to predict the HS side on a test patient which already has the hippocampal segmentation done. It takes approximately 1 minutes to run. @@ -58,12 +62,7 @@ pytest ``` ### Errors -If you run into errors at this stage and need help, you can re-run by changing the last line of the command by the command below to save the terminal outputs in a txt file. Please send `pytest_errors.log` to us so we can work with you to solve any problems. [How best to reach us.](#contact) - -```bash -# run the test -pytest -s | tee -``` +The native installation is not supported as it is very system and software dependant. If you encounter any issues at this stage we recommend to install the docker version. You will find `pytest_errors.log` in the folder where you launched the command. @@ -72,4 +71,4 @@ Please see our [FAQs](https://aid-hs.readthedocs.io/en/latest/FAQs.html) for com ## Contact -If have any question please contact `m.ripart@ucl.ac.uk` for support \ No newline at end of file +If have any question please contact `meld.study@gmail.com` \ No newline at end of file diff --git a/docs/install_singularity.md b/docs/install_singularity.md index 5f4fc08..2620013 100644 --- a/docs/install_singularity.md +++ b/docs/install_singularity.md @@ -1,7 +1,5 @@ # Singularity container -**WARNING: Installation and use not yet tested. Please do let us know if you are succeeding / failing to use the singularity container on HPC** - The Singularity container has been created to be used on HPC supporting Linux as they do not work with Docker container. If you are not working on a HPC, we recommend to install the docker version of container. Notes: @@ -9,7 +7,7 @@ Notes: - You will need **~20GB of space** to install the container - The image contains Miniconda 3, HippUnfold v1.1.0, and AID-HS. The whole image is ~18GB. . -Here is the video tutorial demonstrating how to do the singularity installation - [Docker and Singularity Installation of AID-HS Tutorial](https://www.youtube.com/watch?v=RRAET7r05ys&t=11s&ab_channel=MELDproject). +Here is the video tutorial demonstrating how to do the singularity installation - [Docker and Singularity Installation of AID-HS Tutorial](https://www.youtube.com/watch?v=RRAET7r05ys&t=11s&ab_channel=MELDproject). Note that some part of the installation video are outdated for AID-HS > v1.1.0. Please follow the github guidelines for up to date instructions. ## Prerequisites @@ -29,31 +27,38 @@ Make sure you have 20GB of storage space available for the container singularity build aidhs.sif docker://meldproject/aidhs:latest ``` +## AID-HS license +In order to run AID-HS you need to have a `aidhs_license.txt` in the aidhs folder. To get this file, please fill out the [AID-HS registration form](https://docs.google.com/forms/d/e/1FAIpQLSdPbtraBZ2s0HD1W8qtF11wr_fYVTWZjraED03Rtl2ZjxeRMA/viewform?usp=header). Once submitted, your application will be automatically reviewed and the aidhs_license.txt file will be send to your email. + ## Installation & configuration -Before being able to use AID-HS on your data, data paths need to be set up and the pretrained model needs to be downloaded. -1. Make sure you have 1GB available for the aidhs data. -2. Download and unzip the aidhs_data_folder by running: -```bash -wget https://figshare.com/ndownloader/files/54145361?private_link=48c92b1b53f8f0c67dec --output-document aidhs_data_folder.tar.xz -tar xf aidhs_data_folder.tar.xz -``` +Before being able to use AID-HS on your data, data paths need to be set up and the pretrained model needs to be downloaded. + +1. Make sure you have 20GB of storage space available for the docker, and 1GB available for the aidhs data. +2. Create the **aidhs_data** folder, if it doesn't exist already. This folder is where where you would like to store MRI data to run the tool. 3. Run this command to set the paths needed: -- : Add the path to aidhs_data_folder +- : Add the path to aidhs_data folder +- : path where the `aidhs_license.txt` has been saved + ```bash -export SINGULARITY_BINDPATH=/rds/project/kw350/rds-kw350-meld/test_aidhs/aidhs_data_folder/:/data +export SINGULARITY_BINDPATH=/:/data,/aidhs_license.txt:/aidhs_license.txt:ro +export SINGULARITYENV_AIDHS_LICENSE=/aidhs_license.txt ``` OR with Apptainer ```bash -export APPTAINER_BINDPATH=/rds/project/kw350/rds-kw350-meld/test_aidhs/aidhs_data_folder/:/data +export APPTAINER_BINDPATH=/:/data,/aidhs_license.txt:/aidhs_license.txt:ro +export APPTAINERENV_AIDHS_LICENSE=/aidhs_license.txt ``` -## Download AID-HS container & Verify installation -The line below will download AID-HS in the folder where you ran the command, and then run a test to verify that everything is installed and set up properly. It may take up to an 1h to download the singularity image and then takes approximately 1 minute to run the test. - +:::{admonition} Singularity +:class: tip +You can add those paths to your `~/.bashrc` file to ensure they are always activated when opening a new terminal. +::: +4. Run this command to download the data folder ```bash -singularity exec aidhs.sif /bin/bash -c "cd /app && pytest" +singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/prepare_aidhs.py " ``` +It will download the data in the aidhs_data folder you set up in step 2. ### Errors @@ -73,4 +78,4 @@ You will find `pytest_errors.log` in the folder where you launched the command. Please see our [FAQ page](https://aid-hs.readthedocs.io/en/latest/FAQs.html) for common installation problems and questions ## Contact -If you encounter any errors, please contact `m.ripart@ucl.ac.uk` for support +If you encounter any errors, please contact `meld.stydy@gmail.com` for support diff --git a/docs/run_prediction_pipeline.md b/docs/run_prediction_pipeline.md index dcb73be..73c4273 100644 --- a/docs/run_prediction_pipeline.md +++ b/docs/run_prediction_pipeline.md @@ -111,7 +111,8 @@ You can tune the AID-HS pipeline command using additional variables and flags as | **Mandatory variables** | Comment | |-------|---| |either ```-id ``` | if you want to run the pipeline on 1 single subject.| -|or ```-ids ``` | if you want to run the pipeline on more than 1 subject, you can pass the name of a text file containing the list of subjects. An example 'subjects_list.txt' is provided in the . | +|or ```-ids ``` | if you want to run the pipeline on more than 1 subject, you can pass the name of a text file containing the list of subjects. An example 'subjects_list.txt' is provided in the . | +|```-demos ```| The name of the csv file containing the demographic information as detailled in the [guidelines](https://aid-hs.readthedocs.io/en/latest/prepare_data.html#prepare-the-mri-data-in-bids-format-mandatory) and [provided demographic information](https://aid-hs.readthedocs.io/en/latest/prepare_data.html#prepare-the-demographic-information-to-run-the-harmonisation). An example 'demographics_file.csv' is provided in the .| | **Optional variables** | | ```-harmo_code ``` | provide the harmonisation code if you want to harmonise your data before prediction. This requires to have [computed the harmonisation parameters](https://aid-hs.readthedocs.io/en/latest/harmonisation.html) beforehand. The harmonisation code should start with H, e.g. H1. | |```--parallelise``` | use this flag to speed up the segmentation by running HippUnfold on multiple subjects in parallel. | @@ -132,7 +133,7 @@ To run the whole prediction pipeline on subject 'test001' without harmonising th :sync: Docker Linux ```bash -DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 +DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -demos demographics_file.csv ``` ::: @@ -140,7 +141,7 @@ DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_pati :sync: Docker Windows ```bash -docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 +docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -demos demographics_file.csv ``` ::: @@ -148,7 +149,7 @@ docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipelin :sync: Singularity ```bash -singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001" +singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -demos demographics_file.csv" ``` ::: @@ -156,7 +157,7 @@ singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_p :sync: native ```bash -python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 +python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -demos demographics_file.csv ``` ::: @@ -169,7 +170,7 @@ To run the whole prediction pipeline on subject 'test001' using harmonisation co :sync: Docker Linux ```bash -DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1 +DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1 -demos demographics_file.csv ``` ::: @@ -177,7 +178,7 @@ DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_pati :sync: Docker Windows ```bash -docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1 +docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1 -demos demographics_file.csv ``` ::: @@ -185,14 +186,14 @@ docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipelin :sync: Singularity ```bash -singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1" +singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1 -demos demographics_file.csv" ``` ::: :::{tab-item} Native :sync: native ```bash -python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1 +python scripts/new_patient_pipeline/new_patient_pipeline.py -id sub-test001 -harmo_code H1 -demos demographics_file.csv ``` ::: @@ -204,14 +205,14 @@ To run the whole prediction pipeline on multiples subjects with parallelisation: :::{tab-item} Docker Linux :sync: Docker linux ```bash -DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt --parallelise +DOCKER_USER="$(id -u):$(id -g)" docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt -demos demographics_file.csv --parallelise ``` ::: :::{tab-item} Docker Windows/Mac :sync: Docker Windows ```bash -docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt --parallelise +docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt -demos demographics_file.csv --parallelise ``` ::: @@ -219,7 +220,7 @@ docker compose run aidhs python scripts/new_patient_pipeline/new_patient_pipelin :sync: Singularity ```bash -singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt --parallelise" +singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt -demos demographics_file.csv --parallelise" ``` ::: @@ -227,7 +228,7 @@ singularity exec aidhs.sif /bin/bash -c "cd /app && python scripts/new_patient_p :sync: native ```bash -python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt --parallelise +python scripts/new_patient_pipeline/new_patient_pipeline.py -ids list_subjects.txt -demos demographics_file.csv --parallelise ``` ::: diff --git a/scripts/manage_results/create_prediction_report.py b/scripts/manage_results/create_prediction_report.py index 41c7da6..5a64257 100644 --- a/scripts/manage_results/create_prediction_report.py +++ b/scripts/manage_results/create_prediction_report.py @@ -335,8 +335,12 @@ def plot_segmentations_subject(subject, hippunfold_folder, output_file, hemis=[' file_hipo_seg = os.path.join(hippunfold_folder, f'sub-{subject}', 'anat', f'sub-{subject}_hemi-{hemi}_space-cropT1w_desc-subfields_atlas-bigbrain_dseg.nii.gz') #load images - img_array = sitk.GetArrayFromImage(sitk.ReadImage(file_hipo, sitk.sitkFloat32)) # convert to sitk object - seg_array = sitk.GetArrayFromImage(sitk.ReadImage(file_hipo_seg, sitk.sitkInt64)) + img = sitk.ReadImage(file_hipo, sitk.sitkFloat32) + seg = sitk.ReadImage(file_hipo_seg, sitk.sitkInt64) + img = sitk.DICOMOrient(img, 'RAS') # reorient to RAS to make sure plotting works as expected + seg = sitk.DICOMOrient(seg, 'RAS') + img_array = sitk.GetArrayFromImage(img) + seg_array = sitk.GetArrayFromImage(seg) # get cmap _ , cmap_labels = return_labels_cmap() @@ -345,7 +349,7 @@ def plot_segmentations_subject(subject, hippunfold_folder, output_file, hemis=[' img_array = np.rot90(img_array, 2) seg_array = np.rot90(seg_array, 2) x,y,z = coords_seg_extract(seg_array) - axs[0,i].imshow(img_array[x,:,:], cmap='gray') + axs[0,i].imshow(img_array[x,:,:], origin='lower', cmap='gray') axs[0,i].imshow(seg_array[x,:,:], origin='lower', cmap=cmap_labels, alpha=0.4) axs[0,i].axis('off') axs[0,i].text(1, 125, 'R', fontsize = 8, color='red') @@ -380,9 +384,11 @@ def plot_segmentations_subject(subject, hippunfold_folder, output_file, hemis=[' fig.savefig(output_file, dpi=96, transparent =True) -def create_surf_plot(borders, faces, vertices, cmap=False, file='./tmp.png'): +def create_surf_plot(borders, faces, vertices, cmap=False, file=None): """plot and reload surface images""" from hippunfold_toolbox import plotting + if file is None: + file = tempfile.NamedTemporaryFile(suffix='.png').name fig, ax_temp = plt.subplots(nrows=1, ncols=1, figsize=(8,8), subplot_kw={'projection': "3d"}) plotting.surfplot_cdata(ax_temp,borders,faces,vertices, cmap=cmap) ax_temp.view_init(elev=90, azim=-90) diff --git a/scripts/new_patient_pipeline/new_patient_pipeline.py b/scripts/new_patient_pipeline/new_patient_pipeline.py index b0d9e01..15c245f 100644 --- a/scripts/new_patient_pipeline/new_patient_pipeline.py +++ b/scripts/new_patient_pipeline/new_patient_pipeline.py @@ -2,13 +2,13 @@ import argparse import sys import time - +import shutil from scripts.new_patient_pipeline.run_pipeline_segmentation import run_pipeline_segmentation from scripts.preprocess.extract_features_hdf5 import extract_features_hdf5 from scripts.new_patient_pipeline.run_pipeline_preprocessing import run_pipeline_preprocessing from scripts.new_patient_pipeline.run_pipeline_prediction import run_pipeline_prediction from aidhs.tools_pipeline import get_m -from aidhs.paths import DATA_PATH, BASE_PATH, FS_SUBJECTS_PATH, HIPPUNFOLD_SUBJECTS_PATH, BIDS_SUBJECTS_PATH +from aidhs.paths import DATA_PATH, BASE_PATH, FS_SUBJECTS_PATH, HIPPUNFOLD_SUBJECTS_PATH, BIDS_SUBJECTS_PATH, DEMOGRAPHIC_FEATURES_FILE class Logger(object): def __init__(self, sys_type=sys.stdout, filename='AIDHS_output.log'): @@ -49,9 +49,9 @@ def flush(self): ) parser.add_argument('-demos', '--demographic_file', type=str, - help='provide the demographic files for the harmonisation', + help='provide the name of the demographic files for the harmonisation (e.g. demographics_file.csv). this file should be placed in the main aidhs_data folder', required=False, - default=None, + default='demographics_file.csv', ) parser.add_argument('--harmo_only', action="store_true", @@ -80,13 +80,33 @@ def flush(self): args = parser.parse_args() print(args) - + + #--------------------------------------------------------------------------------- + ### Test aidhs license exists + from aidhs.test.test_aidhs_license import test_license + test_license() + #--------------------------------------------------------------------------------- ### CHECKS if (args.harmo_only) & (args.list_ids == None): print('ERROR: Please provide a list of subjects for the harmonisation. We recommend 20 subjects') os.sys.exit(-1) + #--------------------------------------------------------------------------------- + ### Create tmp demographic file + demographic_file_tmp = DEMOGRAPHIC_FEATURES_FILE + if args.demographic_file is None: + demographic_file = os.path.join(DATA_PATH, "demographics_file.csv") + if not os.path.isfile(demographic_file): + print(f'ERROR: No demographic file found in default {demographic_file}.Please provide a demographic file.') + os.sys.exit(-1) + else: + demographic_file = os.path.join(DATA_PATH, args.demographic_file) + if not os.path.isfile(demographic_file): + print(f'ERROR: Could not find a demographic at {demographic_file}.Please ensure the demographic file is placed in the main aidhs_data folder and only the name of the demographic file is parsed (e.g. demographics_file.csv)') + os.sys.exit(-1) + shutil.copy(demographic_file, demographic_file_tmp) + #--------------------------------------------------------------------------------- ### SEGMENTATION ### diff --git a/scripts/new_patient_pipeline/prepare_aidhs.py b/scripts/new_patient_pipeline/prepare_aidhs.py new file mode 100644 index 0000000..76e2f6a --- /dev/null +++ b/scripts/new_patient_pipeline/prepare_aidhs.py @@ -0,0 +1,108 @@ +import argparse +import os +from configparser import ConfigParser, NoOptionError +import sys +import shutil +import importlib + + +def prepare_config(): + # get scripts dir (parent dir of dir that this file is in) + SCRIPTS_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + # read config file from scripts_dir + config_fname = os.path.join(SCRIPTS_DIR, 'config.ini') + + def copy_config(): + print("Creating new config.ini from config.ini.example") + shutil.copy(os.path.join(SCRIPTS_DIR, "config.ini.example"), config_fname) + + def get_yn_input(): + r = input() + while r.lower() not in ('y', 'n'): + r = input("Unknown value. Either input y or n:\n") + if r.lower() == 'y': + return True + return False + + def get_path_input(): + p = input("Please enter the full path to your aidhs data folder:\n") + p = os.path.abspath(p) + if not os.path.isdir(p): + print(f"{p} is not a valid directory") + return get_path_input() + return p + + # create config file + if os.path.isfile(config_fname): + config = ConfigParser() + config.read(config_fname) + print(f"Found existing config.ini in {config_fname}") + try: + # check that all relevant items are defined in config + config.get("DEFAULT", "data_path") + config.get("develop", "base_path") + config.get("develop", "experiment_path") + except NoOptionError as e: + print("Existing config.ini is not in the right format. Would you like to recreate it now? (y/n)") + if get_yn_input(): + copy_config() + else: + print("Exiting without setting up config.ini.") + sys.exit() + current_data_path = config.get("DEFAULT", "data_path") + #default path /data doesn't work on mac, so ensure they reset if it doesn't exist. + if not os.path.isdir(current_data_path) and not ("KEEP_DATA_PATH" in os.environ): + print(f'The current aidhs data folder path, {config.get("DEFAULT", "data_path")} does not exist. Please reset it now.') + else: + print(f'The current aidhs data folder path is {config.get("DEFAULT", "data_path")}. Would you like to change it? (y/n)') + if ("KEEP_DATA_PATH" in os.environ) or (not get_yn_input()): + print("Leaving aidhs data folder unchanged.") + return + else: + copy_config() + + # fill in aidhs_data_path + aidhs_data_path = get_path_input() + config = ConfigParser() + config.read(config_fname) + config.set("DEFAULT", "data_path", aidhs_data_path) + with open(config_fname, "w") as configfile: + config.write(configfile) + print(f"Successfully changed aidhs data folder to {aidhs_data_path}") + +if __name__ == '__main__': + # ensure that all data is downloaded + parser = argparse.ArgumentParser(description="Setup the classifier: Create config.ini and download test data and pre-trained models") + parser.add_argument('--skip-config', action="store_true", help="do not create config.ini" ) + parser.add_argument('--skip-download-data', action = "store_true", help="do not attempt to download test data") + parser.add_argument("--force-download", action="store_true", help="download data even if exists already") + parser.add_argument("--update_test", action="store_true", help="only update the test data") + args = parser.parse_args() + + from aidhs.download_data import download_aidhs_data, check_data + + #--------------------------------------------------------------------------------- + ### Test aidhs license exists + from aidhs.test.test_aidhs_license import test_license + test_license() + #--------------------------------------------------------------------------------- + + # create and populate config.ini + if not args.skip_config: + prepare_config() + + # need to do this import here, because above we are setting up the config.ini + # which is read when using aidhs_classifier.paths + from aidhs import paths + importlib.reload(paths) + from aidhs.paths import DATA_PATH + + if not args.skip_download_data: + # check that data don't already exist + check_data(args.force_download) + print(f"Downloading aidhs data into {DATA_PATH}") + download_aidhs_data(DATA_PATH) + print("Done.") + else: + print("Skip downloading aidhs data") + \ No newline at end of file diff --git a/scripts/new_patient_pipeline/run_pipeline_prediction.py b/scripts/new_patient_pipeline/run_pipeline_prediction.py index 603ec5a..8615fc2 100644 --- a/scripts/new_patient_pipeline/run_pipeline_prediction.py +++ b/scripts/new_patient_pipeline/run_pipeline_prediction.py @@ -15,9 +15,9 @@ import pickle from PIL import Image import tempfile - +import shutil from aidhs.aidhs_cohort_hip import AidhsCohort, AidhsSubject -from aidhs.paths import DATA_PATH, HIPPUNFOLD_SUBJECTS_PATH, SUBFIELDS_LABEL_FILE, EXPERIMENT_PATH, PARAMS_PATH +from aidhs.paths import DATA_PATH, HIPPUNFOLD_SUBJECTS_PATH, SUBFIELDS_LABEL_FILE, EXPERIMENT_PATH, PARAMS_PATH, DEMOGRAPHIC_FEATURES_FILE from aidhs.train_evaluate import create_dataset_file, predict_subject from aidhs.tools_pipeline import get_m import aidhs.hippunfold_plotting as plotting @@ -406,15 +406,19 @@ def plot_segmentations_subject(subject, hippunfold_folder, output_file, hemis=[' file_hipo_seg = os.path.join(hippunfold_folder, 'hippunfold', subject_bids, 'anat', f'{subject_bids}_hemi-{hemi}_space-cropT1w_desc-subfields_atlas-bigbrain_dseg.nii.gz') #load images - img_array = sitk.GetArrayFromImage(sitk.ReadImage(file_hipo, sitk.sitkFloat32)) # convert to sitk object - seg_array = sitk.GetArrayFromImage(sitk.ReadImage(file_hipo_seg, sitk.sitkInt64)) + img = sitk.ReadImage(file_hipo, sitk.sitkFloat32) + seg = sitk.ReadImage(file_hipo_seg, sitk.sitkInt64) + img = sitk.DICOMOrient(img, 'RAS') # reorient to RAS to make sure plotting works as expected + seg = sitk.DICOMOrient(seg, 'RAS') + img_array = sitk.GetArrayFromImage(img) + seg_array = sitk.GetArrayFromImage(seg) # get cmap _ , cmap_labels = return_labels_cmap() # plot on conventional coronal view x,y,z = coords_seg_extract(seg_array) - axs[0,i].imshow(np.fliplr(img_array[:,y,:]), cmap='gray') + axs[0,i].imshow(np.fliplr(img_array[:,y,:]), origin='lower', cmap='gray') axs[0,i].imshow(np.fliplr(seg_array[:,y,:]), origin='lower', cmap=cmap_labels, alpha=0.4) axs[0,i].axis('off') axs[0,i].text(1, 125, 'R', fontsize = 8, color='red') @@ -443,8 +447,10 @@ def plot_segmentations_subject(subject, hippunfold_folder, output_file, hemis=[' fig.savefig(output_file, dpi=96, transparent =True) -def create_surf_plot(borders, faces, vertices, cmap=False, file='/tmp/tmp.png'): +def create_surf_plot(borders, faces, vertices, cmap=False, file=None): """plot and reload surface images""" + if file is None: + file = tempfile.NamedTemporaryFile(suffix='.png').name fig, ax_temp = plt.subplots(nrows=1, ncols=1, figsize=(8,8), subplot_kw={'projection': "3d"}) plotting.surfplot_cdata(ax_temp,borders,faces,vertices, cmap=cmap) ax_temp.view_init(elev=90, azim=-90) @@ -875,6 +881,12 @@ def run_pipeline_prediction(list_ids=None, sub_id=None, harmo_code='noHarmo', ve help="Harmonisation code", required=False, ) + parser.add_argument('-demos', '--demographic_file', + type=str, + help='provide the name of the demographic files for the harmonisation (e.g. demographics_file.csv). this file should be placed in the main aidhs_data folder', + required=False, + default='demographics_file.csv', + ) parser.add_argument("--debug_mode", help="mode to debug error", required=False, @@ -884,7 +896,29 @@ def run_pipeline_prediction(list_ids=None, sub_id=None, harmo_code='noHarmo', ve args = parser.parse_args() print(args) - + + #--------------------------------------------------------------------------------- + ### Test aidhs license exists + from aidhs.test.test_aidhs_license import test_license + test_license() + + #--------------------------------------------------------------------------------- + ### Create tmp demographic file + demographic_file_tmp = DEMOGRAPHIC_FEATURES_FILE + if args.demographic_file is None: + demographic_file = os.path.join(DATA_PATH, "demographics_file.csv") + if not os.path.isfile(demographic_file): + print(f'ERROR: No demographic file found in default {demographic_file}.Please provide a demographic file.') + os.sys.exit(-1) + else: + demographic_file = os.path.join(DATA_PATH, args.demographic_file) + if not os.path.isfile(demographic_file): + print(f'ERROR: Could not find a demographic at {demographic_file}.Please ensure the demographic file is placed in the main aidhs_data folder and only the name of the demographic file is parsed (e.g. demographics_file.csv)') + os.sys.exit(-1) + shutil.copy(demographic_file, demographic_file_tmp) + + #--------------------------------------------------------------------------------- + run_pipeline_prediction( list_ids=args.list_ids, sub_id=args.id, diff --git a/scripts/new_patient_pipeline/run_pipeline_preprocessing.py b/scripts/new_patient_pipeline/run_pipeline_preprocessing.py index 9f56f54..770a3ae 100755 --- a/scripts/new_patient_pipeline/run_pipeline_preprocessing.py +++ b/scripts/new_patient_pipeline/run_pipeline_preprocessing.py @@ -13,7 +13,7 @@ import time import tempfile import argparse - +import shutil from aidhs.aidhs_cohort_hip import AidhsCohort from aidhs.data_preprocessing import Preprocess, Feature from aidhs.paths import DATA_PATH, BASE_PATH, PARAMS_PATH, ICV_PARAMS_FILE, NORM_CONTROLS_PARAMS_FILE, COMBAT_PARAMS_FILE, CLIPPING_PARAMS_FILE, DEMOGRAPHIC_FEATURES_FILE @@ -308,6 +308,12 @@ def run_pipeline_preprocessing(harmo_code, list_ids=None, sub_id=None, output_di help="harmonisation code", required=False, ) + parser.add_argument('-demos', '--demographic_file', + type=str, + help='provide the name of the demographic files for the harmonisation (e.g. demographics_file.csv). this file should be placed in the main aidhs_data folder', + required=False, + default='demographics_file.csv', + ) parser.add_argument('--harmo_only', action="store_true", help='only compute the harmonisation combat parameters, no further process', @@ -324,7 +330,28 @@ def run_pipeline_preprocessing(harmo_code, list_ids=None, sub_id=None, output_di args = parser.parse_args() print(args) - + + #--------------------------------------------------------------------------------- + ### Test aidhs license exists + from aidhs.test.test_aidhs_license import test_license + test_license() + + #--------------------------------------------------------------------------------- + ### Create tmp demographic file + demographic_file_tmp = DEMOGRAPHIC_FEATURES_FILE + if args.demographic_file is None: + demographic_file = os.path.join(DATA_PATH, "demographics_file.csv") + if not os.path.isfile(demographic_file): + print(f'ERROR: No demographic file found in default {demographic_file}.Please provide a demographic file.') + os.sys.exit(-1) + else: + demographic_file = os.path.join(DATA_PATH, args.demographic_file) + if not os.path.isfile(demographic_file): + print(f'ERROR: Could not find a demographic at {demographic_file}.Please ensure the demographic file is placed in the main aidhs_data folder and only the name of the demographic file is parsed (e.g. demographics_file.csv)') + os.sys.exit(-1) + shutil.copy(demographic_file, demographic_file_tmp) + #--------------------------------------------------------------------------------- + run_pipeline_preprocessing( harmo_code=args.harmo_code, list_ids=args.list_ids, diff --git a/scripts/new_patient_pipeline/run_pipeline_segmentation.py b/scripts/new_patient_pipeline/run_pipeline_segmentation.py index dafc846..0a3edde 100644 --- a/scripts/new_patient_pipeline/run_pipeline_segmentation.py +++ b/scripts/new_patient_pipeline/run_pipeline_segmentation.py @@ -7,11 +7,12 @@ import pandas as pd import json import bids.layout +import shutil from aidhs.tools_pipeline import get_m from scripts.preprocess.run_script_segmentation import run_hippunfold_parallel, run_hippunfold from scripts.preprocess.run_script_dataprep import prepare_T1_parallel ,prepare_T1, bidsify_results, extract_surface_features -from aidhs.paths import DATA_PATH, FS_SUBJECTS_PATH, HIPPUNFOLD_SUBJECTS_PATH, BIDS_SUBJECTS_PATH +from aidhs.paths import DATA_PATH, FS_SUBJECTS_PATH, HIPPUNFOLD_SUBJECTS_PATH, BIDS_SUBJECTS_PATH, DEMOGRAPHIC_FEATURES_FILE def init(lock): global starting @@ -248,6 +249,12 @@ def run_pipeline_segmentation(list_ids=None, sub_id=None, input_dir=None, fs_dir help="File containing list of ids. Can be txt or csv with 'ID' column", required=False, ) + parser.add_argument('-demos', '--demographic_file', + type=str, + help='provide the name of the demographic files for the harmonisation (e.g. demographics_file.csv). this file should be placed in the main aidhs_data folder', + required=False, + default='demographics_file.csv', + ) parser.add_argument("--parallelise", help="parallelise segmentation", required=False, @@ -274,6 +281,28 @@ def run_pipeline_segmentation(list_ids=None, sub_id=None, input_dir=None, fs_dir # initialise parameters use_fastsurfer = True + #--------------------------------------------------------------------------------- + ### Test aidhs license exists + from aidhs.test.test_aidhs_license import test_license + test_license() + + #--------------------------------------------------------------------------------- + ### Create tmp demographic file + demographic_file_tmp = DEMOGRAPHIC_FEATURES_FILE + if args.demographic_file is None: + demographic_file = os.path.join(DATA_PATH, "demographics_file.csv") + if not os.path.isfile(demographic_file): + print(f'ERROR: No demographic file found in default {demographic_file}.Please provide a demographic file.') + os.sys.exit(-1) + else: + demographic_file = os.path.join(DATA_PATH, args.demographic_file) + if not os.path.isfile(demographic_file): + print(f'ERROR: Could not find a demographic at {demographic_file}.Please ensure the demographic file is placed in the main aidhs_data folder and only the name of the demographic file is parsed (e.g. demographics_file.csv)') + os.sys.exit(-1) + shutil.copy(demographic_file, demographic_file_tmp) + + #--------------------------------------------------------------------------------- + run_pipeline_segmentation(list_ids=args.list_ids, sub_id=args.id, input_dir=input_dir, diff --git a/scripts/preprocess/run_script_segmentation.py b/scripts/preprocess/run_script_segmentation.py index 65ffd55..57d8777 100644 --- a/scripts/preprocess/run_script_segmentation.py +++ b/scripts/preprocess/run_script_segmentation.py @@ -43,7 +43,8 @@ def run_hippunfold_parallel(subjects, bids_dir=None, hippo_dir=None, num_procs=1 if subjects_to_run!=[]: print(get_m(f'Start Hippunfold segmentation in parallel for {subjects_to_run}', None, 'INFO')) - command = format(f"hippunfold {bids_dir} {hippo_dir} participant --participant-label {' '.join(subjects_to_run)} --core {num_procs} --modality T1w") + subjects_to_run_shortformat = [subject_id.split('sub-')[-1] for subject_id in subjects_to_run] + command = format(f"hippunfold {bids_dir} {hippo_dir} participant --participant-label {' '.join(subjects_to_run_shortformat)} --core {num_procs} --modality T1w") if verbose: print(command) proc = Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding='utf-8') @@ -55,8 +56,10 @@ def run_hippunfold_parallel(subjects, bids_dir=None, hippo_dir=None, num_procs=1 if delete_intermediate: print(get_m(f'Delete intermediate files that takes lot of space: hippunfold_outputs/hippunfold//warps and hippunfold_outputs/work folders', subjects_to_run, 'INFO')) for subject in subjects_to_run: - shutil.rmtree(hippo_dir, 'hippunfold', subject, 'warps') - shutil.rmtree(hippo_dir, 'work', f'{subject}_work.tar.gz') + if os.path.exists(os.path.join(hippo_dir, 'hippunfold', subject, 'warps')): + shutil.rmtree(os.path.join(hippo_dir, 'hippunfold', subject, 'warps')) + if os.path.isfile(os.path.join(hippo_dir, 'work', f'{subject_bids_id}_work.tar.gz')): + os.remove(os.path.join(hippo_dir, 'work', f'{subject_bids_id}_work.tar.gz')) return True else: print(get_m(f'Hippunfold segmentation failed for 1 of the subject. Please check the logs at {hippo_dir}/logs/', None, 'ERROR')) diff --git a/setup.py b/setup.py index 94eb1b3..2162374 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="aidhs", - version="1.0.1", + version="1.1.0", packages=find_packages(), package_dir={"aidhs": "aidhs"}, )