Support Multimodal datasets

Author: nikita-smetanin

src/together/utils/files.py

@@ -159,12 +159,6 @@ def _check_conversation_type(
                     line_number=idx + 1,
                     error_source="key_value",
                 )
-            if not isinstance(message[column], str):
-                raise InvalidFileFormatError(
-                    message=f"Column `{column}` is not a string on line {idx + 1}. Found {type(message[column])}",
-                    line_number=idx + 1,
-                    error_source="text_field",
-                )
 
 
 def _check_conversation_roles(

tests/unit/test_files_checks.py

@@ -41,6 +41,39 @@ def test_check_jsonl_valid_instruction(tmp_path: Path):
     assert report["has_min_samples"]
 
 
+def test_check_jsonl_valid_instruction_multimodal(tmp_path: Path):
+    file = tmp_path / "valid_instruction_multimodal.jsonl"
+    content = [
+        {
+            "prompt": [
+                {
+                    "type": "text",
+                    "text": "What's the difference between these two images?",
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/jpeg;base64,..."},
+                },
+                {
+                    "type": "image_url",
+                    "image_url": {"url": "data:image/jpeg;base64,..."},
+                },
+            ],
+            "completion": "The first image is a cat, the second image is a dog.",
+        },
+    ]
+
+    with file.open("w") as f:
+        f.write("\n".join(json.dumps(item) for item in content))
+
+    report = check_file(file)
+
+    assert report["is_check_passed"]
+    assert report["utf8"]
+    assert report["num_samples"] == len(content)
+    assert report["has_min_samples"]
+
+
 def test_check_jsonl_valid_conversational_single_turn(tmp_path: Path):
     # Create a valid JSONL file with conversational format and 1 user-assistant turn pair
     file = tmp_path / "valid_conversational_single_turn.jsonl"
@@ -120,6 +153,48 @@ def test_check_jsonl_valid_conversational_multiple_turns(tmp_path: Path):
     assert report["has_min_samples"]
 
 
+def test_check_jsonl_valid_conversational_multimodal_single_turn(tmp_path: Path):
+    file = tmp_path / "valid_conversational_multimodal_single_turn.jsonl"
+    content = [
+        {
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "What's the difference between these two images?",
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "data:image/jpeg;base64,..."},
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "data:image/jpeg;base64,..."},
+                        },
+                    ],
+                },
+                {
+                    "role": "assistant",
+                    "content": [{"type": "text", "text": "Hi there!"}],
+                },
+            ]
+        },
+    ]
+
+    with file.open("w") as f:
+        f.write("\n".join(json.dumps(item) for item in content))
+
+    report = check_file(file)
+
+    print(report)
+    assert report["is_check_passed"]
+    assert report["utf8"]
+    assert report["num_samples"] == len(content)
+    assert report["has_min_samples"]
+
+
 def test_check_jsonl_empty_file(tmp_path: Path):
     # Create an empty JSONL file
     file = tmp_path / "empty.jsonl"