change to use synapse defaults

Author: andrewelamb

synapseclient/extensions/curator/file_based_metadata_task.py

@@ -36,9 +36,6 @@
     "boolean": ColumnType.BOOLEAN_LIST,
 }
 
-MAX_LIST_STRING_ITEM_SIZE = 100
-MAX_LIST_LENGTH = 50
-
 
 def create_json_schema_entity_view(
     syn: Synapse,
@@ -223,19 +220,7 @@ def _create_synapse_column_from_js_property(
         A Synapse Column based on the JSON Schema property.
     """
     column_type = _get_column_type_from_js_property(js_property)
-    maximum_size = None
-    maximum_list_length = None
-    if column_type in LIST_TYPE_DICT.values():
-        maximum_list_length = MAX_LIST_LENGTH
-        if column_type == ColumnType.STRING_LIST:
-            maximum_size = MAX_LIST_STRING_ITEM_SIZE
-
-    return Column(
-        name=name,
-        column_type=column_type,
-        maximum_size=maximum_size,
-        maximum_list_length=maximum_list_length,
-    )
+    return Column(name=name, column_type=column_type)
 
 
 def _get_column_type_from_js_property(js_property: dict[str, Any]) -> ColumnType:

tests/unit/synapseclient/extensions/test_file_based_metadata_task.py

@@ -12,6 +12,7 @@
 import shutil
 import tempfile
 import unittest
+from typing import Any
 from unittest.mock import Mock, mock_open, patch
 
 import pandas as pd
@@ -26,6 +27,7 @@
 )
 from synapseclient.extensions.curator.file_based_metadata_task import (
     _create_columns_from_json_schema,
+    _create_synapse_column_from_js_property,
     _get_column_type_from_js_one_of_list,
     _get_column_type_from_js_property,
     _get_list_column_type_from_js_property,
@@ -50,7 +52,7 @@
     SchemaRegistryColumnConfig,
     get_latest_schema_uri,
 )
-from synapseclient.models import ColumnType
+from synapseclient.models import Column, ColumnType
 from synapseclient.models.curation import (
     FileBasedMetadataTaskProperties,
     RecordBasedMetadataTaskProperties,
@@ -1670,50 +1672,6 @@ def test_create_columns_from_json_schema_success(self):
         assert all(hasattr(col, "name") for col in columns)
         assert all(hasattr(col, "column_type") for col in columns)
 
-    def test_get_column_type_from_js_property_enum(self):
-        """Test getting column type for enum property."""
-        # GIVEN a JSON schema property with an enum
-        js_property = {"enum": ["option1", "option2", "option3"]}
-
-        # WHEN I get the column type
-        result = _get_column_type_from_js_property(js_property)
-
-        # THEN it should return STRING type
-        assert result == ColumnType.STRING
-
-    def test_get_column_type_from_js_property_array(self):
-        """Test getting column type for array property."""
-        # GIVEN a JSON schema property with array type
-        js_property = {"type": "array", "items": {"type": "string"}}
-
-        # WHEN I get the column type
-        result = _get_column_type_from_js_property(js_property)
-
-        # THEN it should return a list type
-        assert result == ColumnType.STRING_LIST
-
-    def test_get_column_type_from_js_property_one_of(self):
-        """Test getting column type for oneOf property."""
-        # GIVEN a JSON schema property with oneOf
-        js_property = {"oneOf": [{"type": "string"}, {"type": "null"}]}
-
-        # WHEN I get the column type
-        result = _get_column_type_from_js_property(js_property)
-
-        # THEN it should return STRING type
-        assert result == ColumnType.STRING
-
-    def test_get_column_type_from_js_property_fallback(self):
-        """Test getting column type fallback to STRING."""
-        # GIVEN a JSON schema property without recognizable type
-        js_property = {"description": "some property"}
-
-        # WHEN I get the column type
-        result = _get_column_type_from_js_property(js_property)
-
-        # THEN it should return STRING type as fallback
-        assert result == ColumnType.STRING
-
     def test_get_column_type_from_js_one_of_list_with_enum(self):
         """Test getting column type from oneOf list containing enum."""
         # GIVEN a oneOf list with an enum
@@ -1722,8 +1680,8 @@ def test_get_column_type_from_js_one_of_list_with_enum(self):
         # WHEN I get the column type
         result = _get_column_type_from_js_one_of_list(js_one_of_list)
 
-        # THEN it should return STRING type
-        assert result == ColumnType.STRING
+        # THEN it should return MEDIUMTEXT type
+        assert result == ColumnType.MEDIUMTEXT
 
     def test_get_column_type_from_js_one_of_list_single_type(self):
         """Test getting column type from oneOf list with single non-null type."""
@@ -1758,8 +1716,8 @@ def test_get_column_type_from_js_one_of_list_fallback(self):
         # WHEN I get the column type
         result = _get_column_type_from_js_one_of_list(js_one_of_list)
 
-        # THEN it should return STRING type as fallback
-        assert result == ColumnType.STRING
+        # THEN it should return MEDIUMTEXT type as fallback
+        assert result == ColumnType.MEDIUMTEXT
 
     def test_get_list_column_type_from_js_property_with_enum(self):
         """Test getting list column type for property with enum items."""
@@ -1795,6 +1753,120 @@ def test_get_list_column_type_from_js_property_fallback(self):
         assert result == ColumnType.STRING_LIST
 
 
+@pytest.mark.parametrize(
+    "schema, expected",
+    [
+        (
+            {
+                "properties": {
+                    "string_col": {"type": "string"},
+                }
+            },
+            [
+                Column(name="string_col", column_type=ColumnType.MEDIUMTEXT),
+            ],
+        ),
+        (
+            {
+                "properties": {
+                    "string_col": {"type": "string"},
+                    "int_col": {"type": "integer"},
+                    "bool_col": {"type": "boolean"},
+                }
+            },
+            [
+                Column(name="string_col", column_type=ColumnType.MEDIUMTEXT),
+                Column(name="int_col", column_type=ColumnType.INTEGER),
+                Column(name="bool_col", column_type=ColumnType.BOOLEAN),
+            ],
+        ),
+    ],
+    ids=["one column", "three columns"],
+)
+def test_create_columns_from_json_schema(
+    schema: dict[str, Any], expected: list[Column]
+):
+    assert _create_columns_from_json_schema(schema) == expected
+
+
+@pytest.mark.parametrize(
+    "prop, name, expected_type",
+    [
+        (
+            {"type": "array", "items": {"type": "string"}},
+            "string_list_col",
+            ColumnType.STRING_LIST,
+        ),
+        (
+            {"type": "array", "items": {"type": "integer"}},
+            "int_list_col",
+            ColumnType.INTEGER_LIST,
+        ),
+        (
+            {"type": "array", "items": {"type": "boolean"}},
+            "bool_list_col",
+            ColumnType.BOOLEAN_LIST,
+        ),
+        (
+            {"type": "string"},
+            "string_col",
+            ColumnType.MEDIUMTEXT,
+        ),
+    ],
+    ids=["string_list", "integer_list", "boolean_list", "string"],
+)
+def test_create_synapse_column_from_js_property(
+    prop: dict[str, Any], name: str, expected_type: ColumnType
+):
+    result = _create_synapse_column_from_js_property(prop, name)
+    assert isinstance(result, Column)
+    assert result.name == name
+    assert result.column_type == expected_type
+
+
+@pytest.mark.parametrize(
+    "schema",
+    [{}, {"properties": []}],
+    ids=["empty schema", "properties is not a d ict"],
+)
+def test_create_columns_from_json_schema_exceptions(schema: dict[str, Any]):
+    with pytest.raises(ValueError):
+        _create_columns_from_json_schema(schema)
+
+
+@pytest.mark.parametrize(
+    "prop, expected",
+    [
+        ({"enum": ["a", "b", "c"]}, ColumnType.MEDIUMTEXT),
+        ({"type": "string"}, ColumnType.MEDIUMTEXT),
+        ({"type": "integer"}, ColumnType.INTEGER),
+        ({"type": "number"}, ColumnType.DOUBLE),
+        ({"type": "boolean"}, ColumnType.BOOLEAN),
+        ({"type": ["integer", "null"]}, ColumnType.INTEGER),
+        ({"type": ["integer", "string"]}, ColumnType.MEDIUMTEXT),
+        ({"type": "array", "items": {"type": "integer"}}, ColumnType.INTEGER_LIST),
+        ({"oneOf": [{"type": "integer"}, {"type": "null"}]}, ColumnType.INTEGER),
+        ({"type": "unknown"}, ColumnType.MEDIUMTEXT),
+        ({}, ColumnType.MEDIUMTEXT),
+    ],
+    ids=[
+        "enum_property",
+        "type_string",
+        "type_integer",
+        "type_number",
+        "type_boolean",
+        "type_list_nullable",
+        "type_list_multiple_types",
+        "type_array",
+        "one_of_list",
+        "unknown_type",
+        "empty_property",
+    ],
+)
+def test_get_column_type_from_js_property(prop: dict[str, Any], expected: ColumnType):
+    assert _get_column_type_from_js_property(prop) == expected
+
+
 class TestGetLatestSchemaUri(unittest.TestCase):
     """Test cases for get_latest_schema_uri function."""