Add dataset reference to dataset decorator (openproblems-bio#500)

scottgigante-immunai · web-flow · commit d20f2fd60074 · 2022-07-21T08:56:15.000-04:00
* add dataset reference

* add reference to multimodal data

* add DOIs for zebrafish, pancreas, immmune

* refactor and add reference to 10x 1k

* add missing references

* document addition to loader decorator
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -256,8 +256,8 @@ def f2(adata):
 
 Datasets are loaded under `openproblems/data`. Each data loading function should
 download the appropriate dataset from a stable location (e.g. from Figshare) be
-decorated with `openproblems.data.utils.loader(data_url="https://data.link")` in order
-to cache the result.
+decorated with `openproblems.data.utils.loader(data_url="https://data.link",
+data_reference="https://doi.org/10.0/123")` in order to cache the result.
 
 Data should be provided in a raw count format. We assume that `adata.X` contains the raw
 (count) data for the primary modality; this will also be copied to
diff --git a/openproblems/data/Wagner_2018_zebrafish_embryo_CRISPR.py b/openproblems/data/Wagner_2018_zebrafish_embryo_CRISPR.py
@@ -4,7 +4,10 @@
 import scprep
 
 
-@utils.loader(data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE112294")
+@utils.loader(
+    data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE112294",
+    data_reference="https://doi.org/10.1126/science.aar4362",
+)
 def load_zebrafish_chd_tyr(test=False):
     """Download zebrafish data from GEO accession GSE112294"""
 
diff --git a/openproblems/data/cengen.py b/openproblems/data/cengen.py
@@ -11,7 +11,9 @@
 )
 
 
-@utils.loader(data_url=URL)
+@utils.loader(
+    data_url=URL, data_reference="https://doi.org/10.1016/j.neuron.2018.07.042"
+)
 def load_cengen(test=False):
     """Download CeNGEN data from GitHub.
 
diff --git a/openproblems/data/human_blood_nestorowa2016.py b/openproblems/data/human_blood_nestorowa2016.py
@@ -8,7 +8,9 @@
 URL = "https://ndownloader.figshare.com/files/25555751"
 
 
-@utils.loader(data_url=URL)
+@utils.loader(
+    data_url=URL, data_reference="https://doi.org/10.1182/blood-2016-05-716480"
+)
 def load_human_blood_nestorowa2016(test=False):
     """Download Nesterova data from Figshare."""
     if test:
diff --git a/openproblems/data/immune_cells.py b/openproblems/data/immune_cells.py
@@ -8,7 +8,7 @@
 URL = "https://ndownloader.figshare.com/files/25717328"
 
 
-@utils.loader(data_url=URL)
+@utils.loader(data_url=URL, data_reference="https://doi.org/10.1038/s41592-021-01336-8")
 def load_immune(test=False):
     """Download immune human data from figshare."""
     if test:
diff --git a/openproblems/data/mouse_blood_olssen_labelled.py b/openproblems/data/mouse_blood_olssen_labelled.py
@@ -8,7 +8,7 @@
 URL = "https://ndownloader.figshare.com/files/27346712"
 
 
-@utils.loader(data_url=URL)
+@utils.loader(data_url=URL, data_reference="https://doi.org/10.1038/nature19348")
 def load_olsson_2016_mouse_blood(test=False):
     """Download Olsson, 2016_mouse_blood, Nature, 2016 data from Figshare."""
     if test:
diff --git a/openproblems/data/multimodal/citeseq.py b/openproblems/data/multimodal/citeseq.py
@@ -15,7 +15,10 @@
 )
 
 
-@loader(data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE100866")
+@loader(
+    data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE100866",
+    data_reference="https://doi.org/10.1038/nmeth.4380",
+)
 def load_citeseq_cbmc(test=False):
     """Download CITEseq data from GEO."""
     if test:
diff --git a/openproblems/data/multimodal/sample.py b/openproblems/data/multimodal/sample.py
@@ -11,7 +11,10 @@
 import scipy.sparse
 
 
-@loader(data_url="https://openproblems.bio")
+@loader(
+    data_url="https://openproblems.bio",
+    data_reference="https://github.com/openproblems-bio/openproblems",
+)
 def load_sample_data(test=True):
     """Create a simple dataset to use for testing in multimodal applications."""
     assert test
diff --git a/openproblems/data/multimodal/scicar/base.py b/openproblems/data/multimodal/scicar/base.py
@@ -6,6 +6,8 @@
 import scprep
 import tempfile
 
+DATA_REFERENCE = "https://doi.org/10.1126/science.aau0730"
+
 
 def load_scicar(
     rna_url,
diff --git a/openproblems/data/multimodal/scicar/cell_lines.py b/openproblems/data/multimodal/scicar/cell_lines.py
@@ -1,5 +1,6 @@
 from ...utils import loader
 from ..utils import subset_joint_data
+from .base import DATA_REFERENCE
 from .base import load_scicar
 
 rna_url = (
@@ -28,7 +29,10 @@
 )
 
 
-@loader(data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089")
+@loader(
+    data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089",
+    data_reference=DATA_REFERENCE,
+)
 def load_scicar_cell_lines(test=False):
     """Download sci-CAR cell lines data from GEO."""
     if test:
diff --git a/openproblems/data/multimodal/scicar/mouse_kidney.py b/openproblems/data/multimodal/scicar/mouse_kidney.py
@@ -1,5 +1,6 @@
 from ...utils import loader
 from ..utils import subset_joint_data
+from .base import DATA_REFERENCE
 from .base import load_scicar
 
 rna_url = (
@@ -28,7 +29,10 @@
 )
 
 
-@loader(data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089")
+@loader(
+    data_url="https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE117089",
+    data_reference=DATA_REFERENCE,
+)
 def load_scicar_mouse_kidney(test=False):
     """Download sci-CAR mouse kidney data from GEO."""
     if test:
diff --git a/openproblems/data/pancreas.py b/openproblems/data/pancreas.py
@@ -8,7 +8,7 @@
 URL = "https://ndownloader.figshare.com/files/24539828"
 
 
-@utils.loader(data_url=URL)
+@utils.loader(data_url=URL, data_reference="https://doi.org/10.1038/s41592-021-01336-8")
 def load_pancreas(test=False):
     """Download pancreas data from figshare."""
     if test:
diff --git a/openproblems/data/pbmc.py b/openproblems/data/pbmc.py
diff --git a/openproblems/data/sample.py b/openproblems/data/sample.py
@@ -8,7 +8,10 @@
 import scipy.sparse
 
 
-@loader(data_url="https://openproblems.bio")
+@loader(
+    data_url="https://openproblems.bio",
+    data_reference="https://github.com/openproblems-bio/openproblems",
+)
 def load_sample_data(test=True):
     """Create a simple dataset to use for testing in multimodal applications."""
     assert test
diff --git a/openproblems/data/tabula_muris_senis.py b/openproblems/data/tabula_muris_senis.py
@@ -59,7 +59,10 @@ def make_anndata_list(subset_df, test):
     return adata_list
 
 
-@utils.loader(data_url="https://tabula-muris-senis.ds.czbiohub.org/")
+@utils.loader(
+    data_url="https://tabula-muris-senis.ds.czbiohub.org/",
+    data_reference="https://doi.org/10.1038/s41586-020-2496-1",
+)
 def load_tabula_muris_senis(test=False, method_list=None, organ_list=None):
     """Load tubula_muris_senis datasets into 1 anndata object based on user input.
 
diff --git a/openproblems/data/tenx.py b/openproblems/data/tenx.py
@@ -5,10 +5,29 @@
 import scprep
 import tempfile
 
-URL = "https://ndownloader.figshare.com/files/25555739"
+PBMC_1K_URL = "https://ndownloader.figshare.com/files/24974582"
+PBMC_5K_URL = "https://ndownloader.figshare.com/files/25555739"
+REFERENCE_URL = "https://www.10xgenomics.com/resources/datasets"
 
 
-@utils.loader(data_url=URL)
+@utils.loader(data_url=PBMC_1K_URL, data_reference=REFERENCE_URL)
+def load_tenx_1k_pbmc(test=False):
+    """Download PBMC data from Figshare."""
+    if test:
+        adata = load_tenx_1k_pbmc(test=False)
+        sc.pp.subsample(adata, n_obs=100)
+        adata = adata[:, :1000]
+        utils.filter_genes_cells(adata)
+    else:
+        with tempfile.TemporaryDirectory() as tempdir:
+            filepath = os.path.join(tempdir, "pbmc.h5ad")
+            scprep.io.download.download_url(PBMC_1K_URL, filepath)
+            adata = sc.read_h5ad(filepath)
+            utils.filter_genes_cells(adata)
+    return adata
+
+
+@utils.loader(data_url=PBMC_5K_URL, data_reference=REFERENCE_URL)
 def load_tenx_5k_pbmc(test=False):
     """Download 5k PBMCs from 10x Genomics."""
     if test:
@@ -30,7 +49,7 @@ def load_tenx_5k_pbmc(test=False):
     else:
         with tempfile.TemporaryDirectory() as tempdir:
             filepath = os.path.join(tempdir, "10x_5k_pbmc.h5ad")
-            scprep.io.download.download_url(URL, filepath)
+            scprep.io.download.download_url(PBMC_5K_URL, filepath)
             adata = sc.read(filepath)
 
             adata.var_names_make_unique()
diff --git a/openproblems/data/utils.py b/openproblems/data/utils.py
@@ -33,13 +33,15 @@ def _cache_path(func, *args, **kwargs):
     return os.path.join(TEMPDIR, filename)
 
 
-def loader(data_url):
+def loader(data_url, data_reference):
     """Decorate a data loader function.
 
     Parameters
     ----------
     data_url : str
         Link to the original source of the dataset
+    data_reference : str
+        Link to the paper describing how the dataset was generated
     """
 
     def decorator(func):
@@ -71,7 +73,7 @@ def apply_func(*args, **kwargs):
                 adata.write_h5ad(filepath)
                 return adata
 
-        apply_func.metadata = dict(data_url=data_url)
+        apply_func.metadata = dict(data_url=data_url, data_reference=data_reference)
         return apply_func
 
     return decorator
diff --git a/openproblems/data/zebrafish.py b/openproblems/data/zebrafish.py
@@ -10,7 +10,7 @@
 )
 
 
-@utils.loader(data_url=URL)
+@utils.loader(data_url=URL, data_reference="https://doi.org/10.1126/science.aar4362")
 def load_zebrafish(test=False):
     """Download zebrafish data from figshare."""
     with tempfile.TemporaryDirectory() as tempdir:
diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/datasets/immune.py b/openproblems/tasks/_batch_integration/batch_integration_graph/datasets/immune.py
@@ -7,6 +7,7 @@
 @dataset(
     dataset_name="Immune (by batch)",
     data_url=load_immune.metadata["data_url"],
+    data_reference=load_immune.metadata["data_reference"],
     dataset_summary="TODO",
     image="openproblems",
 )
diff --git a/openproblems/tasks/_batch_integration/batch_integration_graph/datasets/pancreas.py b/openproblems/tasks/_batch_integration/batch_integration_graph/datasets/pancreas.py
@@ -7,6 +7,7 @@
 @dataset(
     dataset_name="Pancreas (by batch)",
     data_url=load_pancreas.metadata["data_url"],
+    data_reference=load_pancreas.metadata["data_reference"],
     dataset_summary="TODO",
     image="openproblems",
 )
diff --git a/openproblems/tasks/denoising/datasets/pbmc.py b/openproblems/tasks/denoising/datasets/pbmc.py
@@ -1,15 +1,16 @@
-from ....data.pbmc import load_pbmc
+from ....data.tenx import load_tenx_1k_pbmc
 from ....tools.decorators import dataset
 from . import utils
 
 
 @dataset(
     "1k PBMCs from a Healthy Donor (10x/v3)",
-    data_url=load_pbmc.metadata["data_url"],
+    data_url=load_tenx_1k_pbmc.metadata["data_url"],
+    data_reference=load_tenx_1k_pbmc.metadata["data_reference"],
     dataset_summary="TODO",
     image="openproblems-python-extras",
 )
 def pbmc(test=False):
-    adata = load_pbmc(test=test)
+    adata = load_tenx_1k_pbmc(test=test)
     adata = utils.split_data(adata)
     return adata
diff --git a/openproblems/tasks/dimensionality_reduction/datasets/human_blood_nestorowa2016.py b/openproblems/tasks/dimensionality_reduction/datasets/human_blood_nestorowa2016.py
@@ -5,6 +5,7 @@
 @dataset(
     "Human blood (HSCs and differentiation thereof). Nestorowa, et al. Blood. 2016",
     data_url=load_human_blood_nestorowa2016.metadata["data_url"],
+    data_reference=load_human_blood_nestorowa2016.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def human_blood_nestorowa2016(test=False):
diff --git a/openproblems/tasks/dimensionality_reduction/datasets/mouse_blood_olssen_labelled.py b/openproblems/tasks/dimensionality_reduction/datasets/mouse_blood_olssen_labelled.py
@@ -5,6 +5,7 @@
 @dataset(
     "Mouse blood. Olsson, et al. Nature. 2016",
     data_url=load_olsson_2016_mouse_blood.metadata["data_url"],
+    data_reference=load_olsson_2016_mouse_blood.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def olsson_2016_mouse_blood(test=False):
diff --git a/openproblems/tasks/dimensionality_reduction/datasets/tenx_5k_pbmc.py b/openproblems/tasks/dimensionality_reduction/datasets/tenx_5k_pbmc.py
@@ -1,11 +1,12 @@
-from ....data.tenx_5k_pbmc import load_tenx_5k_pbmc
+from ....data.tenx import load_tenx_5k_pbmc
 from ....tools.decorators import dataset
 
 
 @dataset(
     "5k Peripheral blood mononuclear cells (PBMCs) from a healthy donor. "
     "10x Genomics; July 24, 2019.",
     data_url=load_tenx_5k_pbmc.metadata["data_url"],
+    data_reference=load_tenx_5k_pbmc.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def tenx_5k_pbmc(test=False):
diff --git a/openproblems/tasks/label_projection/datasets/cengen.py b/openproblems/tasks/label_projection/datasets/cengen.py
@@ -7,6 +7,7 @@
 @dataset(
     "CeNGEN (by batch)",
     data_url=load_cengen.metadata["data_url"],
+    data_reference=load_cengen.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def cengen_batch(test=False):
@@ -27,6 +28,7 @@ def cengen_batch(test=False):
 @dataset(
     "CeNGEN (random split)",
     data_url=load_cengen.metadata["data_url"],
+    data_reference=load_cengen.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def cengen_random(test=False):
diff --git a/openproblems/tasks/label_projection/datasets/pancreas.py b/openproblems/tasks/label_projection/datasets/pancreas.py
@@ -8,6 +8,7 @@
 @dataset(
     "Pancreas (by batch)",
     data_url=load_pancreas.metadata["data_url"],
+    data_reference=load_pancreas.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def pancreas_batch(test=False):
@@ -28,6 +29,7 @@ def pancreas_batch(test=False):
 @dataset(
     "Pancreas (random split)",
     data_url=load_pancreas.metadata["data_url"],
+    data_reference=load_pancreas.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def pancreas_random(test=False):
@@ -46,6 +48,7 @@ def pancreas_random(test=False):
 @dataset(
     r"Pancreas (random split with 20% label noise",
     data_url=load_pancreas.metadata["data_url"],
+    data_reference=load_pancreas.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def pancreas_random_label_noise(test=False):
diff --git a/openproblems/tasks/label_projection/datasets/tabula_muris_senis.py b/openproblems/tasks/label_projection/datasets/tabula_muris_senis.py
@@ -7,6 +7,7 @@
 @dataset(
     "Tabula Muris Senis Lung (random split)",
     data_url=load_tabula_muris_senis.metadata["data_url"],
+    data_reference=load_tabula_muris_senis.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def tabula_muris_senis_lung_random(test=False):
diff --git a/openproblems/tasks/label_projection/datasets/zebrafish.py b/openproblems/tasks/label_projection/datasets/zebrafish.py
@@ -7,6 +7,7 @@
 @dataset(
     "Zebrafish (by labels)",
     data_url=load_zebrafish.metadata["data_url"],
+    data_reference=load_zebrafish.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def zebrafish_labels(test=False):
@@ -20,6 +21,7 @@ def zebrafish_labels(test=False):
 @dataset(
     "Zebrafish (random split)",
     data_url=load_zebrafish.metadata["data_url"],
+    data_reference=load_zebrafish.metadata["data_reference"],
     dataset_summary="TODO",
 )
 def zebrafish_random(test=False):
diff --git a/openproblems/tasks/multimodal_data_integration/datasets/citeseq.py b/openproblems/tasks/multimodal_data_integration/datasets/citeseq.py
diff --git a/openproblems/tasks/multimodal_data_integration/datasets/scicar.py b/openproblems/tasks/multimodal_data_integration/datasets/scicar.py
diff --git a/openproblems/tasks/regulatory_effect_prediction/datasets/scicar.py b/openproblems/tasks/regulatory_effect_prediction/datasets/scicar.py
diff --git a/openproblems/tools/decorators.py b/openproblems/tools/decorators.py
diff --git a/test/test_3_datasets.py b/test/test_3_datasets.py
diff --git a/workflow/parse_nextflow.py b/workflow/parse_nextflow.py

Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,9 @@`
`11`	`11`	`)`
`12`	`12`
`13`	`13`
`14`		`-@utils.loader(data_url=URL)`
	`14`	`+@utils.loader(`
	`15`	`+ data_url=URL, data_reference="https://doi.org/10.1016/j.neuron.2018.07.042"`
	`16`	`+)`
`15`	`17`	`def load_cengen(test=False):`
`16`	`18`	`"""Download CeNGEN data from GitHub.`
`17`	`19`