Merge pull request #377 from MLMI2-CSSI/metadata_error_handling

Metadata error handling

Author: ascourtas

foundry/foundry.py

@@ -4,6 +4,7 @@
 from json2table import convert
 import numpy as np
 import pandas as pd
+from pydantic import ValidationError
 from typing import Any, Dict, List
 import logging
 import warnings
@@ -23,15 +24,16 @@
 from foundry.models import (
     FoundryMetadata,
     FoundryConfig,
-    FoundryDataset
+    FoundryDataset,
+    FoundryBase
 )
 from foundry.https_download import download_file, recursive_ls
 from foundry.https_upload import upload_to_endpoint
 
 logger = logging.getLogger(__name__)
 
 
-class Foundry(FoundryMetadata):
+class Foundry(FoundryBase):
     """Foundry Client Base Class
     TODO:
     -------
@@ -49,7 +51,8 @@ class Foundry(FoundryMetadata):
 
     def __init__(
             self, name=None, no_browser=False, no_local_server=False, index="mdf", authorizers=None,
-            download=True, globus=True, verbose=False, metadata=None, interval=10, **data
+            download=True, globus=True, verbose=False, metadata=None, interval=10,
+            **data
     ):
         """Initialize a Foundry client
         Args:
@@ -75,6 +78,13 @@ def __init__(
         self.index = index
         self.auths = None
 
+        self.config = FoundryConfig(
+            dataframe_file="foundry_dataframe.json",
+            metadata_file="foundry_metadata.json",
+            local=False,
+            local_cache_dir="./data",
+        )
+
         if authorizers:
             self.auths = authorizers
         else:
@@ -392,6 +402,9 @@ def publish_dataset(
             of dataset. Contains `source_id`, which can be used to check the
             status of the submission
         """
+        # ensure metadata is properly formatted
+        self.validate_metadata(foundry_metadata)
+
         # ensure that one of `https_data_path` or `globus_data_source` have been assigned values
         if (https_data_path and globus_data_source) or \
                 (https_data_path is None and globus_data_source is None):
@@ -774,3 +787,27 @@ def to_tensorflow(self, split: str = None):
 
         inputs, targets = self._get_inputs_targets(split)
         return TensorflowSequence(inputs, targets)
+
+    def validate_metadata(self, metadata):
+        """Validate the JSON message against the FoundryMetadata model
+
+        Arguments:
+            metadata (dict): Metadata information provided by the user.
+
+        Raises:
+            ValidationError: if metadata supplied by user does not meet the specificiation of a
+            FoundryMetadata object.
+
+        """
+        try:
+            FoundryMetadata(**metadata)
+            logger.debug("Metadata validation successful!")
+        except ValidationError as e:
+            logger.error("Metadata validation failed!")
+            for error in e.errors():
+                field_name = ".".join([item for item in error['loc'] if isinstance(item, str)])
+                error_description = error['msg']
+                error_message = f"""There is an issue validating the metadata for the field '{field_name}':
+                The error message returned is: '{error_description}'."""
+                logger.error(error_message)
+            raise e

foundry/models.py

@@ -1,5 +1,5 @@
 from typing import List, Dict, Optional, Any
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, StrictInt, StrictStr
 from enum import Enum
 import pandas as pd
 from json2table import convert
@@ -58,6 +58,7 @@ def _repr_html_(self):
 
 # END Classes for Foundry Data Package Specification
 
+
 class FoundryDatasetType(Enum):
     """Foundry Dataset Types
     Enumeration of the possible Foundry dataset types
@@ -70,23 +71,40 @@ class FoundryDatasetType(Enum):
 
 
 class FoundryKeyClass(BaseModel):
-    label: str = ""
-    name: str = ""
+    label: StrictStr = Field(..., description="The label that exists in the data")
+    name: StrictStr = Field(..., description="The name the label maps onto.")
 
 
 class FoundryKey(BaseModel):
-    key: List[str] = []
-    type: str = ""
-    filter: Optional[str] = ""
-    units: Optional[str] = ""
-    description: Optional[str] = ""
+    key: List[StrictStr] = Field(..., description="Column or header name for tabular data, key/path for HDF5 data")
+    type: StrictStr = Field(..., description="Whether input or target")
     classes: Optional[List[FoundryKeyClass]]
+    description: Optional[StrictStr]
+    filter: Optional[StrictStr]
+    units: Optional[StrictStr]
 
 
 class FoundrySplit(BaseModel):
-    type: str = ""
-    path: Optional[str] = ""
-    label: Optional[str] = ""
+    type: StrictStr = Field(..., description="The kind of partition of the dataset (train, test, validation, etc)")
+    path: Optional[StrictStr]
+    label: Optional[StrictStr]
+
+
+class FoundryMetadata(BaseModel):
+    """Foundry Dataset
+    Schema for Foundry Datasets. This includes specifications of inputs, outputs, type, version, and more
+    """
+    data_type: FoundryDatasetType = Field(..., description="The kind of data in the dataset, e.g. tabular, json, hdf5")
+    domain: List[StrictStr] = Field(..., description="The domain of applicability. e.g., materials science, chemistry, machine vision")
+    keys: List[FoundryKey] = Field(..., description="Keys describing how to load the data")
+    dataframe: Optional[Any]
+    n_items: Optional[StrictInt]
+    short_name: Optional[StrictStr]
+    splits: Optional[List[FoundrySplit]]
+    task_type: Optional[List[StrictStr]]
+
+    class Config:
+        arbitrary_types_allowed = True
 
 
 class FoundryDataset(BaseModel):
@@ -134,7 +152,7 @@ def _repr_html_(self):
         return convert(json.loads(self.json()))
 
 
-class FoundryMetadata(BaseModel):
+class FoundryBase(BaseModel):
     dc: Optional[Dict] = {}  # pydantic Datacite?
     mdf: Optional[Dict] = {}
     dataset: FoundryDataset = {}

tests/test_foundry.py

@@ -6,6 +6,7 @@
 from datetime import datetime
 from math import floor
 import numpy as np
+from pydantic import ValidationError
 import requests
 import mdf_toolbox
 import pandas as pd
@@ -134,6 +135,66 @@
     'n_items': 1000
 }
 
+
+pub_test_invalid_metadata = {
+    "keys": [
+        {
+            "key": ["sepal length (cm)"],
+            "type": "input",
+            "units": "cm",
+            "description": 10
+        },
+        {
+            "key": ["sepal width (cm)"],
+            "type": "input",
+            "units": "cm",
+            "description": "sepal width in unit(cm)"
+        },
+        {
+            "key": ["petal length (cm)"],
+            "type": "input",
+            "units": "cm",
+            "description": "petal length in unit(cm)"
+        },
+        {
+            "key": ["petal width (cm)"],
+            "type": "input",
+            "units": "cm",
+            "description": "petal width in unit(cm)"
+        },
+        {
+            "key": ["y"],
+            "type": "output",
+            "units": "",
+            "description": "flower type",
+            "classes": [
+                {
+                    "label": "0",
+                    "name": "setosa"
+                },
+                {
+                    "label": "1",
+                    "name": "versicolor"
+                },
+                {
+                    "label": "2",
+                    "name": "virginica"
+                }
+            ]
+        }
+    ],
+    'splits': [
+        {'label': 'train', 'path': 'train.json', 'type': 'train'},
+        {'label': 'test', 'path': 'test.json', 'type': 'test'}
+    ],
+    "short_name": "example_AS_iris_test_{:.0f}".format(datetime.now().timestamp()),
+    "data_type": "tabular",
+    'task_type': ['unsupervised', 'generative'],
+    'domain': ['materials science', 'chemistry'],
+    'n_items': 1000
+}
+
+
 # Globus endpoint for '_iris_dev' for test publication
 pub_test_data_source = "https://app.globus.org/file-manager?origin_id=e38ee745-6d04-11e5-ba46-22000b92c6ec&origin_path=%2Ffoundry-test%2Firis-dev%2F"
 
@@ -300,6 +361,24 @@ def test_publish_with_https():
     assert res['source_id'] == f"_test_{short_name}_v1.1"
 
 
+def test_publish_invalid_metadata():
+    """Testing the validation of the metadata when publishing data
+    """
+    with pytest.raises(ValidationError) as exc_info:
+        f = Foundry(index="mdf-test", authorizers=auths)
+        timestamp = datetime.now().timestamp()
+        title = "https_publish_test_{:.0f}".format(timestamp)
+        short_name = "https_pub_{:.0f}".format(timestamp)
+        authors = ["A Scourtas"]
+        local_path = "./data/https_test"
+
+        # create test JSON to upload (if it doesn't already exist)
+        _write_test_data(local_path)
+        f.publish_dataset(pub_test_invalid_metadata, title, authors, https_data_path=local_path, short_name=short_name)
+
+    assert exc_info.value.errors()[0]['msg'] == 'str type expected'
+
+
 def test_upload_to_endpoint():
     """Unit test: Test the _upload_to_endpoint() HTTPS functionality on its own, without publishing to MDF
     """