diff --git a/benchmark_utils/image_dataset.py b/benchmark_utils/image_dataset.py deleted file mode 100644 index 524ac74..0000000 --- a/benchmark_utils/image_dataset.py +++ /dev/null @@ -1,33 +0,0 @@ -import os -import random - -from torch.utils.data import Dataset -from typing import Callable -from PIL import Image - - -class ImageDataset(Dataset): - def __init__(self, - folder: str, - transform: Callable = None, - num_images=None): - self.folder = folder - self.transform = transform - self.files = [f for f in os.listdir(folder) if f.endswith(( - '.png', '.jpg', '.jpeg'))] - - if num_images is not None: - self.files.sort() - self.files = random.sample(self.files, num_images) - - def __len__(self): - return len(self.files) - - def __getitem__(self, idx): - img_name = os.path.join(self.folder, self.files[idx]) - image = Image.open(img_name) - - if self.transform: - image = self.transform(image) - - return image diff --git a/config.yml b/config.yml deleted file mode 100644 index e16d5c7..0000000 --- a/config.yml +++ /dev/null @@ -1,5 +0,0 @@ -data_home: /Users/melvinenargeot/Data/benchmark_inverse_problems -data_paths: - generated_datasets: generated_datasets - generated_trainings: generated_training - BSD500: BSD500/BSR/BSDS500/data/images \ No newline at end of file diff --git a/datasets/bsd500_bsd20.py b/datasets/bsd500_bsd20.py deleted file mode 100644 index 26150bf..0000000 --- a/datasets/bsd500_bsd20.py +++ /dev/null @@ -1,118 +0,0 @@ -from benchopt import BaseDataset, safe_import_context, config - -with safe_import_context() as import_ctx: - import deepinv as dinv - import torch - from torchvision import transforms - from benchmark_utils.image_dataset import ImageDataset - from deepinv.physics import Downsampling, Denoising, GaussianNoise - from deepinv.physics.generator import MotionBlurGenerator - - -class Dataset(BaseDataset): - - name = "BSD500_BSD20" - - parameters = { - 'task': ['denoising', - 'gaussian-debluring', - 'motion-debluring', - 'SRx4'], - 'img_size': [256], - } - - requirements = ["datasets"] - - def get_data(self): - # TODO: Remove - device = ( - dinv.utils.get_freer_gpu()) if torch.cuda.is_available() else "cpu" - - n_channels = 3 - - if self.task == "denoising": - noise_level_img = 0.03 - physics = Denoising(GaussianNoise(sigma=noise_level_img)) - elif self.task == "gaussian-debluring": - filter_torch = dinv.physics.blur.gaussian_blur(sigma=(3, 3)) - noise_level_img = 0.03 - n_channels = 3 - - physics = dinv.physics.BlurFFT( - img_size=(n_channels, self.img_size, self.img_size), - filter=filter_torch, - noise_model=dinv.physics.GaussianNoise(sigma=noise_level_img), - device=device - ) - elif self.task == "motion-debluring": - psf_size = 31 - n_channels = 3 - motion_generator = MotionBlurGenerator( - (psf_size, psf_size), - device=device - ) - - filters = motion_generator.step(batch_size=1) - - physics = dinv.physics.BlurFFT( - img_size=(n_channels, self.img_size, self.img_size), - filter=filters["filter"], - device=device - ) - elif self.task == "SRx4": - physics = Downsampling(img_size=(n_channels, - self.img_size, - self.img_size), - filter="bicubic", - factor=4, - device=device) - else: - raise Exception("Unknown task") - - transform = transforms.Compose([ - transforms.Resize((self.img_size, self.img_size)), - transforms.ToTensor() - ]) - - train_dataset = ImageDataset( - config.get_data_path("BSD500") / "train", - transform=transform - ) - - test_dataset = ImageDataset( - config.get_data_path("BSD500") / "val", - transform=transform, - num_images=20 - ) - - dinv_dataset_path = dinv.datasets.generate_dataset( - train_dataset=train_dataset, - test_dataset=test_dataset, - physics=physics, - save_dir=config.get_data_path( - key="generated_datasets" - ) / "bsd500_bsd20", - dataset_filename=self.task, - device=device - ) - - train_dataset = dinv.datasets.HDF5Dataset( - path=dinv_dataset_path, train=True - ) - test_dataset = dinv.datasets.HDF5Dataset( - path=dinv_dataset_path, train=False - ) - - x, y = train_dataset[0] - dinv.utils.plot([x.unsqueeze(0), y.unsqueeze(0)]) - - x, y = test_dataset[0] - dinv.utils.plot([x.unsqueeze(0), y.unsqueeze(0)]) - - return dict( - train_dataset=train_dataset, - test_dataset=test_dataset, - physics=physics, - dataset_name="BSD68", - task_name=self.task - ) diff --git a/datasets/bsd500_cbsd68.py b/datasets/bsd500_cbsd68.py index ccfab17..bdc5e8a 100644 --- a/datasets/bsd500_cbsd68.py +++ b/datasets/bsd500_cbsd68.py @@ -1,11 +1,11 @@ -from benchopt import BaseDataset, safe_import_context, config +from benchopt import BaseDataset, safe_import_context +from benchopt.config import get_data_path with safe_import_context() as import_ctx: import deepinv as dinv import torch from torchvision import transforms from datasets import load_dataset - from benchmark_utils.image_dataset import ImageDataset from benchmark_utils.hugging_face_torch_dataset import ( HuggingFaceTorchDataset ) @@ -77,8 +77,9 @@ def get_data(self): transforms.ToTensor() ]) - train_dataset = ImageDataset( - config.get_data_path("BSD500") / "train", transform=transform + path = get_data_path("BSD500") + train_dataset = dinv.datasets.BSDS500( + path, download=True, transform=transform ) dataset_cbsd68 = load_dataset("deepinv/CBSD68") @@ -90,9 +91,7 @@ def get_data(self): train_dataset=train_dataset, test_dataset=test_dataset, physics=physics, - save_dir=config.get_data_path( - key="generated_datasets" - ) / "bsd500_cbsd68", + save_dir=get_data_path("bsd500_cbsd68"), dataset_filename=self.task, device=device ) diff --git a/datasets/bsd500_imnet100.py b/datasets/bsd500_imnet100.py index 3b277fc..d0f6959 100644 --- a/datasets/bsd500_imnet100.py +++ b/datasets/bsd500_imnet100.py @@ -1,10 +1,10 @@ -from benchopt import BaseDataset, safe_import_context, config +from benchopt import BaseDataset, safe_import_context +from benchopt.config import get_data_path with safe_import_context() as import_ctx: import deepinv as dinv import torch from torchvision import transforms - from benchmark_utils.image_dataset import ImageDataset from benchmark_utils.hugging_face_torch_dataset import ( HuggingFaceTorchDataset ) @@ -77,9 +77,9 @@ def get_data(self): transforms.ToTensor() ]) - train_dataset = ImageDataset( - config.get_data_path("BSD500") / "train", - transform=transform + path = get_data_path("BSD500") + train_dataset = dinv.datasets.BSDS500( + path, download=True, transform=transform ) dataset_miniImnet100 = load_dataset("mterris/miniImnet100") @@ -93,9 +93,7 @@ def get_data(self): train_dataset=train_dataset, test_dataset=test_dataset, physics=physics, - save_dir=config.get_data_path( - key="generated_datasets" - ) / "bsd500_imnet100", + save_dir=get_data_path("bsd500_imnet100"), dataset_filename=self.task, device=device ) diff --git a/datasets/cbsd68_set3c.py b/datasets/cbsd68_set3c.py index 325c160..fadd716 100644 --- a/datasets/cbsd68_set3c.py +++ b/datasets/cbsd68_set3c.py @@ -1,4 +1,5 @@ -from benchopt import BaseDataset, safe_import_context, config +from benchopt import BaseDataset, safe_import_context +from benchopt.config import get_data_path with safe_import_context() as import_ctx: import deepinv as dinv @@ -91,9 +92,7 @@ def get_data(self): train_dataset=train_dataset, test_dataset=test_dataset, physics=physics, - save_dir=config.get_data_path( - key="generated_datasets" - ) / "sbsd68_set3c", + save_dir=get_data_path("cbsd68_set3c"), dataset_filename=self.task, device=device ) diff --git a/test_config.py b/test_config.py index ba1b960..89e5a0f 100644 --- a/test_config.py +++ b/test_config.py @@ -11,3 +11,10 @@ def check_test_solver_install(solver_class): detecting the situation. """ pass + + +def check_test_dataset_get_data(benchmark, dataset_class): + if sys.platform == "darwin": + pytest.skip( + "Skipping test_dataset_get_data on MacOS." + )