Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ public VortexDataSourceV2() {
*
* @param options the data source options containing file paths
* @return the inferred Spark SQL schema
* @throws RuntimeException if required path options are missing
* @throws IllegalArgumentException if no Vortex files can be found under the supplied paths
* @throws RuntimeException if there's an error reading the file or converting the schema
*/
@Override
Expand All @@ -87,7 +87,11 @@ public StructType inferSchema(CaseInsensitiveStringMap options) {
.findFirst();

if (firstFile.isEmpty()) {
throw new RuntimeException(String.format("UNABLE_TO_INFER_SCHEMA format: %s", shortName()));
throw new IllegalArgumentException(String.format(
"Unable to infer schema for %s: no .vortex files found under path %s. "
+ "Check that the path is correct and contains at least one Vortex file, "
+ "or provide an explicit schema.",
shortName(), pathToInfer));
} else {
pathToInfer = firstFile.get();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright the Vortex contributors

package dev.vortex.spark;

import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;

import java.nio.file.Path;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.io.TempDir;

/** Tests for schema inference failure reporting in {@link VortexDataSourceV2}. */
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public final class VortexDataSourceInferSchemaTest {

private SparkSession spark;

@TempDir
Path tempDir;

@BeforeAll
public void setUp() {
spark = SparkSession.builder()
.appName("VortexInferSchemaTest")
.master("local[1]")
.config("spark.driver.host", "127.0.0.1")
.config("spark.ui.enabled", "false")
.getOrCreate();
}

@AfterAll
public void tearDown() {
if (spark != null) {
spark.stop();
}
}

@Test
@DisplayName("Reading a directory without Vortex files reports the offending path")
public void inferSchemaFailureNamesThePath() {
String emptyDir = tempDir.toString();

Throwable thrown = assertThrows(
Throwable.class, () -> spark.read().format("vortex").load(emptyDir));

String allMessages = messagesOf(thrown);
assertTrue(
allMessages.contains("no .vortex files found"),
"error should explain that no Vortex files were found, got: " + allMessages);
assertTrue(allMessages.contains(emptyDir), "error should name the offending path, got: " + allMessages);
}

/** Concatenates the messages of the whole cause chain, since Spark may wrap data source exceptions. */
private static String messagesOf(Throwable thrown) {
StringBuilder sb = new StringBuilder();
for (Throwable t = thrown; t != null; t = t.getCause()) {
sb.append(t.getMessage()).append('\n');
}
return sb.toString();
}
}