Skip to content

Commit 9fe7898

Browse files
marmouremarmoure
authored
Merge pull request #5 from marmoure/feature/load-schema-disk
Feature/load schema disk
2 parents 69dc9a0 + 592948c commit 9fe7898

3 files changed

Lines changed: 92 additions & 100 deletions

File tree

‎src/main/java/com/evolvedbinary/bblValidator/service/CsvValidationService.java‎

Lines changed: 21 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88
import uk.gov.nationalarchives.csv.validator.api.java.*;
99

1010
import java.nio.charset.Charset;
11+
import java.nio.charset.StandardCharsets;
1112
import java.nio.file.Path;
1213
import java.util.ArrayList;
14+
import java.util.Collections;
1315
import java.util.List;
1416

1517
import static uk.gov.nationalarchives.csv.validator.api.CsvValidator$.MODULE$;
@@ -19,39 +21,31 @@
1921
public class CsvValidationService {
2022

2123
private static final Logger LOG = LoggerFactory.getLogger(CsvValidationService.class);
22-
private static final int DEFAULT_MAX_CHARS_PER_CELL = 8096;
2324

2425
@Inject
2526
private SchemaService schemaService;
2627

2728
public ValidationResult validateCsvFile(Path csvFilePath, String schemaId) {
29+
String schemaFilePath = String.valueOf(schemaService.getSchemaFilePath(schemaId));
30+
Charset csvEncoding = StandardCharsets.UTF_8;
31+
boolean validateUtf8Encoding = true;
32+
Charset csvSchemaEncoding = StandardCharsets.UTF_8;
33+
boolean failFast = false;
34+
List<Substitution> pathSubstitutions = Collections.emptyList();
35+
boolean enforceCaseSensitivePathChecks = false;
36+
boolean trace = false;
37+
boolean skipFileChecks = false;
38+
39+
CsvValidatorJavaBridge.ValidationRequest validationRequest = new CsvValidatorJavaBridge.ValidationRequest(
40+
csvFilePath.toString(), csvEncoding, validateUtf8Encoding, schemaFilePath,
41+
csvSchemaEncoding, true, failFast, pathSubstitutions,
42+
enforceCaseSensitivePathChecks, trace, null, skipFileChecks, -1);
43+
2844
long startTime = System.currentTimeMillis();
29-
try {
30-
String schemaFilePath = String.valueOf(schemaService.getSchemaFilePath(schemaId));
31-
Charset csvEncoding = MODULE$.DEFAULT_ENCODING();
32-
boolean validateUtf8Encoding = csvEncoding.name().equals("UTF-8");
33-
Charset csvSchemaEncoding = MODULE$.DEFAULT_ENCODING();
34-
boolean failFast = false;
35-
List<Substitution> pathSubstitutions = new ArrayList<>();
36-
boolean enforceCaseSensitivePathChecks = false;
37-
boolean trace = false;
38-
boolean skipFileChecks = false;
39-
40-
CsvValidatorJavaBridge.ValidationRequest validationRequest = new CsvValidatorJavaBridge.ValidationRequest(
41-
csvFilePath.toString(), csvEncoding, validateUtf8Encoding, schemaFilePath,
42-
csvSchemaEncoding, true, failFast, pathSubstitutions,
43-
enforceCaseSensitivePathChecks, trace, null, skipFileChecks, DEFAULT_MAX_CHARS_PER_CELL);
44-
45-
CsvValidatorJavaBridge.ValidationResult result = CsvValidatorJavaBridge.validate(validationRequest);
46-
List<FailMessage> errors = result.errors();
47-
long executionTime = System.currentTimeMillis() - startTime;
48-
return processValidationMessages(errors, executionTime);
49-
50-
} catch (Exception e) {
51-
long executionTime = System.currentTimeMillis() - startTime;
52-
LOG.error("Error validating CSV file: {}", csvFilePath, e);
53-
return ValidationResult.error("Validation failed: " + e.getMessage(), executionTime);
54-
}
45+
CsvValidatorJavaBridge.ValidationResult result = CsvValidatorJavaBridge.validate(validationRequest);
46+
List<FailMessage> errors = result.errors();
47+
long executionTime = System.currentTimeMillis() - startTime;
48+
return processValidationMessages(errors, executionTime);
5549
}
5650

5751

‎src/main/java/com/evolvedbinary/bblValidator/service/SchemaService.java‎

Lines changed: 68 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22

33
import com.evolvedbinary.bblValidator.dto.SchemaInfo;
44
import com.fasterxml.jackson.databind.ObjectMapper;
5+
import io.micronaut.context.annotation.Value;
56
import jakarta.annotation.PostConstruct;
67
import jakarta.inject.Singleton;
78
import org.slf4j.Logger;
89
import org.slf4j.LoggerFactory;
910

1011
import java.io.IOException;
11-
import java.io.InputStream;
1212
import java.nio.charset.StandardCharsets;
1313
import java.nio.file.Files;
1414
import java.nio.file.Path;
@@ -24,7 +24,9 @@
2424
public class SchemaService {
2525

2626
private static final Logger LOG = LoggerFactory.getLogger(SchemaService.class);
27-
private static final String SCHEMA_DIRECTORY = "schemas";
27+
28+
@Value("${schema.directory}")
29+
private String schemaDirectory;
2830

2931
private final List<SchemaInfo> schemas = new ArrayList<>();
3032
private final Map<String, String> schemaContents = new HashMap<>();
@@ -34,96 +36,90 @@ public class SchemaService {
3436
@PostConstruct
3537
public void loadSchemas() {
3638
try {
37-
// Load schemas from classpath
38-
ClassLoader classLoader = getClass().getClassLoader();
39-
40-
// Get all .json files from the schemas directory
41-
try (InputStream is = classLoader.getResourceAsStream(SCHEMA_DIRECTORY)) {
42-
if (is == null) {
43-
LOG.warn("Schemas directory not found in classpath");
44-
return;
45-
}
39+
// Resolve schema directory path
40+
Path schemaPath = resolveSchemaPath();
41+
42+
if (!Files.exists(schemaPath)) {
43+
LOG.warn("Schemas directory not found: {}", schemaPath);
44+
return;
45+
}
46+
47+
if (!Files.isDirectory(schemaPath)) {
48+
LOG.warn("Schema path is not a directory: {}", schemaPath);
49+
return;
4650
}
4751

4852
// Scan for schema metadata files
49-
loadSchemasFromClasspath();
53+
loadSchemasFromFileSystem(schemaPath);
5054

51-
LOG.info("Loaded {} schemas from disk", schemas.size());
55+
LOG.info("Loaded {} schemas from: {}", schemas.size(), schemaPath);
5256
} catch (Exception e) {
53-
LOG.error("Error loading schemas from disk", e);
57+
LOG.error("Error loading schemas from file system", e);
5458
}
5559
}
5660

57-
private void loadSchemasFromClasspath() {
58-
// close the file input stream
59-
// try with resoucres
60-
// avoid class path
61-
// in application yaml add a property for schema folder
62-
// make it relative to the start up location
63-
// if it starts with a slash then resolve it as absolut path
64-
try {
65-
// Get resource URL and list files
66-
ClassLoader classLoader = getClass().getClassLoader();
67-
var resource = classLoader.getResource(SCHEMA_DIRECTORY);
68-
69-
if (resource != null) {
70-
Path schemaPath = Paths.get(resource.toURI());
71-
72-
try (Stream<Path> paths = Files.walk(schemaPath, 1)) {
73-
paths.filter(path -> path.toString().endsWith(".json"))
74-
.forEach(this::loadSchemaMetadata);
75-
}
76-
}
77-
} catch (Exception e) {
78-
LOG.error("Error scanning schema directory", e);
61+
/**
62+
* Resolves the schema directory path.
63+
* If the path starts with a slash, it's treated as an absolute path.
64+
* Otherwise, it's resolved relative to the current working directory (startup location).
65+
*/
66+
private Path resolveSchemaPath() {
67+
Path schemaPath = Paths.get(schemaDirectory);
68+
if (schemaPath.isAbsolute()) {
69+
// Absolute path
70+
return schemaPath;
71+
} else {
72+
// Relative to current working
73+
Path applicationDir = Paths.get(System.getProperty("user.dir"));
74+
return schemaPath.relativize(applicationDir);
7975
}
8076
}
8177

82-
private void loadSchemaMetadata(Path metadataPath) {
83-
try {
84-
String content = Files.readString(metadataPath, StandardCharsets.UTF_8);
85-
SchemaInfo schemaInfo = objectMapper.readValue(content, SchemaInfo.class);
86-
87-
// Load corresponding schema file
88-
String schemaFileName = metadataPath.getFileName().toString().replace(".json", ".csvs");
89-
Path schemaFilePath = metadataPath.getParent().resolve(schemaFileName);
90-
91-
if (Files.exists(schemaFilePath)) {
92-
String schemaContent = Files.readString(schemaFilePath, StandardCharsets.UTF_8);
93-
schemaContents.put(schemaInfo.getId(), schemaContent);
94-
schemaFilePaths.put(schemaInfo.getId(), schemaFilePath);
95-
schemas.add(schemaInfo);
96-
LOG.debug("Loaded schema: {}", schemaInfo.getId());
97-
} else {
98-
LOG.warn("Schema file not found for metadata: {}", schemaFileName);
99-
}
78+
private void loadSchemasFromFileSystem(Path schemaPath) {
79+
try (Stream<Path> paths = Files.walk(schemaPath, 1)) {
80+
paths.filter(path -> path.toString().endsWith(".json"))
81+
.forEach(path -> {
82+
try {
83+
loadSchemaMetadata(path);
84+
} catch (Exception e) {
85+
LOG.error("Error loading schema metadata from: {}", path, e);
86+
}
87+
});
10088
} catch (IOException e) {
101-
LOG.error("Error loading schema metadata from: {}", metadataPath, e);
89+
LOG.error("Error scanning schema directory: {}", schemaPath, e);
10290
}
10391
}
10492

105-
public List<SchemaInfo> listSchemas() {
106-
return new ArrayList<>(schemas);
107-
}
108-
109-
public String getSchema(String schemaId) throws Exception {
110-
String content = schemaContents.get(schemaId);
111-
112-
if (content == null) {
113-
throw new Exception("Schema not found: " + schemaId);
93+
private void loadSchemaMetadata(Path metadataPath) throws IOException {
94+
String content = Files.readString(metadataPath, StandardCharsets.UTF_8);
95+
SchemaInfo schemaInfo = objectMapper.readValue(content, SchemaInfo.class);
96+
97+
// Load corresponding schema file
98+
String schemaFileName = metadataPath.getFileName().toString().replace(".json", ".csvs");
99+
Path schemaFilePath = metadataPath.getParent().resolve(schemaFileName);
100+
101+
if (Files.exists(schemaFilePath)) {
102+
String schemaContent = Files.readString(schemaFilePath, StandardCharsets.UTF_8);
103+
// TODO: Only load the schema content when needed
104+
schemaContents.put(schemaInfo.getId(), schemaContent);
105+
schemaFilePaths.put(schemaInfo.getId(), schemaFilePath);
106+
schemas.add(schemaInfo);
107+
LOG.debug("Loaded schema: {}", schemaInfo.getId());
108+
} else {
109+
LOG.warn("Schema file not found for metadata: {}", schemaFileName);
114110
}
115-
116-
return content;
117111
}
118112

119-
public Path getSchemaFilePath(String schemaId) throws Exception {
120-
Path filePath = schemaFilePaths.get(schemaId);
113+
public List<SchemaInfo> listSchemas() {
114+
return schemas;
115+
}
121116

122-
if (filePath == null) {
123-
throw new Exception("Schema file path not found: " + schemaId);
124-
}
117+
public String getSchema(String schemaId) {
118+
return schemaContents.get(schemaId);
119+
}
125120

126-
return filePath;
121+
public Path getSchemaFilePath(String schemaId) {
122+
return schemaFilePaths.get(schemaId);
127123
}
128124
}
129125

‎src/main/resources/application.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,6 @@ micronaut:
44
server:
55
port: 8080
66
api:
7-
version: ${project.version}
7+
version: ${project.version}
8+
schema:
9+
directory: schemas

0 commit comments

Comments
 (0)