diff --git a/pom.xml b/pom.xml index fd64d2163..d43661fc7 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 3.20.0 - 1.2.1 + 1.2.2 3.0.3 1.6.3 1.25.5 diff --git a/src/main/java/fr/insee/genesis/Constants.java b/src/main/java/fr/insee/genesis/Constants.java index 5a23c2167..eb06a6197 100644 --- a/src/main/java/fr/insee/genesis/Constants.java +++ b/src/main/java/fr/insee/genesis/Constants.java @@ -2,6 +2,8 @@ import lombok.experimental.UtilityClass; +import java.util.regex.Pattern; + @UtilityClass public class Constants { @@ -11,6 +13,8 @@ public class Constants { public static final String METADATA_SEPARATOR = "."; public static final String DATE_REGEX = "(^([0-9]{4})[\\-\\/]([0-9]|1[0-2]|0[1-9])[\\-\\/]([0-9]|[0-2][0-9]|3[0-1])$)|" + "(^([0-9]|[0-2][0-9]|3[0-1])[\\-\\/]([0-9]|1[0-2]|0[1-9])[\\-\\/]([0-9]{4})$)"; + public static final Pattern QUESTIONNAIRE_ID_PATTERN = + Pattern.compile("^[A-Za-z0-9_-]+$"); public static final String FILTER_RESULT_PREFIX = "FILTER_RESULT_"; public static final String MISSING_SUFFIX = "_MISSING"; public static final String MONGODB_LUNATIC_RAWDATA_COLLECTION_NAME = "lunaticjsondata"; @@ -21,6 +25,8 @@ public class Constants { public static final String MONGODB_CONTEXT_COLLECTION_NAME = "dataProcessingContexts"; public static final String MONGODB_CONTEXTUAL_PREVIOUS_COLLECTION_NAME = "editedPrevious"; public static final String MONGODB_CONTEXTUAL_EXTERNAL_COLLECTION_NAME = "editedExternal"; + public static final String TYPE_PREVIOUS = "PREVIOUS"; + public static final String TYPE_EXTERNAL = "EXTERNAL"; public static final String LOOP_NAME_PREFIX = "BOUCLE"; public static final String MONGODB_RESPONSE_COLLECTION_NAME = "responses"; public static final String MONGODB_RAW_RESPONSES_COLLECTION_NAME = "rawResponses"; diff --git a/src/main/java/fr/insee/genesis/controller/dto/ContextualVariableFileReportDto.java b/src/main/java/fr/insee/genesis/controller/dto/ContextualVariableFileReportDto.java new file mode 100644 index 000000000..b9c6f5ea2 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/dto/ContextualVariableFileReportDto.java @@ -0,0 +1,6 @@ +package fr.insee.genesis.controller.dto; + +public record ContextualVariableFileReportDto( + String fileName, + String type +) {} diff --git a/src/main/java/fr/insee/genesis/controller/dto/SaveContextualVariablesReportDto.java b/src/main/java/fr/insee/genesis/controller/dto/SaveContextualVariablesReportDto.java new file mode 100644 index 000000000..3246288c1 --- /dev/null +++ b/src/main/java/fr/insee/genesis/controller/dto/SaveContextualVariablesReportDto.java @@ -0,0 +1,9 @@ +package fr.insee.genesis.controller.dto; + +import java.util.List; + +public record SaveContextualVariablesReportDto( + String questionnaireId, + int processedFiles, + List files +) {} diff --git a/src/main/java/fr/insee/genesis/controller/rest/responses/ContextualVariableController.java b/src/main/java/fr/insee/genesis/controller/rest/responses/ContextualVariableController.java index ff3a17101..b2a8b0757 100644 --- a/src/main/java/fr/insee/genesis/controller/rest/responses/ContextualVariableController.java +++ b/src/main/java/fr/insee/genesis/controller/rest/responses/ContextualVariableController.java @@ -2,6 +2,7 @@ import fr.insee.genesis.Constants; import fr.insee.genesis.configuration.Config; +import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.ports.api.ContextualExternalVariableApiPort; import fr.insee.genesis.domain.ports.api.ContextualPreviousVariableApiPort; @@ -22,6 +23,9 @@ import java.io.IOException; import java.nio.file.Path; +import java.util.regex.Pattern; + +import static fr.insee.genesis.Constants.QUESTIONNAIRE_ID_PATTERN; @RequestMapping(path = "/contextual-variables") @Controller @@ -34,6 +38,7 @@ public class ContextualVariableController { private final ContextualVariableApiPort contextualVariableApiPort; private final Config config; + @Operation(summary = "Get contextual variables (external and previous)") @GetMapping(path = "/") @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')") @@ -52,16 +57,35 @@ public ResponseEntity getContextualVariables( public ResponseEntity saveContextualVariables( @RequestParam("questionnaireId") String questionnaireId ) throws GenesisException{ - FileUtils fileUtils = new FileUtils(config); + validateQuestionnaireId(questionnaireId); - String contextualFolderPath = fileUtils.getDataFolder(questionnaireId, "WEB", null) + Constants.CONTEXTUAL_FOLDER; + FileUtils fileUtils = new FileUtils(config); - int fileCount = contextualVariableApiPort.saveContextualVariableFiles(questionnaireId, fileUtils,contextualFolderPath); + int fileCount = contextualVariableApiPort.saveContextualVariableFiles(questionnaireId, fileUtils); return ResponseEntity.ok("%d file(s) processed for questionnaire %s !".formatted(fileCount, questionnaireId)); } + @Operation(summary = "Save all contextual variables json files and return processed file details") + @PostMapping(path = "/json/report") + @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER')") + public ResponseEntity saveContextualVariablesWithReport( + @RequestParam("questionnaireId") String questionnaireId + ) throws GenesisException { + validateQuestionnaireId(questionnaireId); + + FileUtils fileUtils = new FileUtils(config); + + SaveContextualVariablesReportDto report = + contextualVariableApiPort.saveContextualVariableFilesWithReport( + questionnaireId, + fileUtils + ); + + return ResponseEntity.ok(report); + } + @Operation(summary = "Add contextual previous json file") @PostMapping(path = "previous/json") @PreAuthorize("hasAnyRole('USER_PLATINE','SCHEDULER','USER_BACK_OFFICE')") @@ -121,4 +145,10 @@ private static void moveFile(String questionnaireId, Mode mode, FileUtils fileUt throw new GenesisException(HttpStatus.INTERNAL_SERVER_ERROR, "Error while moving file to done"); } } + + private void validateQuestionnaireId(String questionnaireId) throws GenesisException { + if (!QUESTIONNAIRE_ID_PATTERN.matcher(questionnaireId).matches()) { + throw new GenesisException(HttpStatus.BAD_REQUEST, "Invalid questionnaireId"); + } + } } diff --git a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java index 152321ee5..f27bb29ce 100644 --- a/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java +++ b/src/main/java/fr/insee/genesis/domain/ports/api/ContextualVariableApiPort.java @@ -1,10 +1,15 @@ package fr.insee.genesis.domain.ports.api; +import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto; import fr.insee.genesis.domain.model.contextualvariable.ContextualVariableModel; import fr.insee.genesis.exceptions.GenesisException; import fr.insee.genesis.infrastructure.utils.FileUtils; public interface ContextualVariableApiPort { ContextualVariableModel getContextualVariable(String collectionInstrumentId, String interrogationId); - int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils, String contextualFolderPath) throws GenesisException; + int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils) throws GenesisException; + SaveContextualVariablesReportDto saveContextualVariableFilesWithReport( + String collectionInstrumentId, + FileUtils fileUtils + ) throws GenesisException; } \ No newline at end of file diff --git a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java index 50e17cef4..8af5bde8a 100644 --- a/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java +++ b/src/main/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonService.java @@ -1,5 +1,7 @@ package fr.insee.genesis.domain.service.contextualvariable; +import fr.insee.genesis.controller.dto.ContextualVariableFileReportDto; +import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto; import fr.insee.genesis.controller.dto.VariableQualityToolDto; import fr.insee.genesis.controller.dto.VariableStateDto; import fr.insee.genesis.domain.model.contextualvariable.ContextualExternalVariableModel; @@ -26,8 +28,13 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Stream; +import static fr.insee.genesis.Constants.CONTEXTUAL_FOLDER; +import static fr.insee.genesis.Constants.TYPE_EXTERNAL; +import static fr.insee.genesis.Constants.TYPE_PREVIOUS; + @Service @Slf4j public class ContextualVariableJsonService implements ContextualVariableApiPort { @@ -76,20 +83,47 @@ public ContextualVariableModel getContextualVariable(String collectionInstrument } @Override - public int saveContextualVariableFiles(String collectionInstrumentId, FileUtils fileUtils, String contextualFolderPath) throws GenesisException { - int fileCount = 0; + public int saveContextualVariableFiles( + String collectionInstrumentId, + FileUtils fileUtils + ) throws GenesisException { + return saveContextualVariableFilesWithReport( + collectionInstrumentId, + fileUtils + ).processedFiles(); + } + + /** + * Generates a detailed report of processed contextual variable files. + * Added to preserve the existing behavior of {@link #saveContextualVariableFiles(String, FileUtils)}, + * which only returns the number of processed files. + */ + @Override + public SaveContextualVariablesReportDto saveContextualVariableFilesWithReport( + String collectionInstrumentId, + FileUtils fileUtils + ) throws GenesisException { + List files = new ArrayList<>(); + String contextualFolderPath = + fileUtils.getDataFolder(collectionInstrumentId, "WEB", null) + + CONTEXTUAL_FOLDER; for (Mode mode : Mode.values()) { - try (Stream filePaths = Files.list(Path.of(contextualFolderPath))) { + Path contextualFolder = Path.of(contextualFolderPath); + try (Stream filePaths = Files.list(contextualFolder)) { Iterator it = filePaths .filter(path -> path.toString().endsWith(".json")) .iterator(); + while (it.hasNext()) { Path jsonFilePath = it.next(); - if (processContextualVariableFile(collectionInstrumentId, jsonFilePath)) { - //If the file is indeed a contextual variables file and had been processed + + Optional report = + processContextualVariableFileForReport(collectionInstrumentId, jsonFilePath); + + if (report.isPresent()) { moveFile(collectionInstrumentId, mode, fileUtils, jsonFilePath.toString()); - fileCount++; + files.add(report.get()); } } } catch (NoSuchFileException nsfe) { @@ -98,9 +132,64 @@ public int saveContextualVariableFiles(String collectionInstrumentId, FileUtils log.warn(ioe.toString()); } } - return fileCount; + + return new SaveContextualVariablesReportDto( + collectionInstrumentId, + files.size(), + files + ); } + private Optional processContextualVariableFileForReport( + String collectionInstrumentId, + Path jsonFilePath + ) throws GenesisException { + try { + Optional type = processContextualVariableFileAndGetType( + collectionInstrumentId, + jsonFilePath + ); + + return type.map(value -> new ContextualVariableFileReportDto( + jsonFilePath.getFileName().toString(), + value + )); + } catch (GenesisException e) { + throw new GenesisException( + e.getStatus(), + "Error while processing file '%s' : %s" + .formatted(jsonFilePath.getFileName().toString(), e.getMessage()) + ); + } + } + + private Optional processContextualVariableFileAndGetType( + String collectionInstrumentId, + Path jsonFilePath + ) throws GenesisException { + boolean isPrevious = contextualPreviousVariableApiPort.readContextualPreviousFile( + collectionInstrumentId.toUpperCase(), + null, + jsonFilePath.toString() + ); + + if (isPrevious) { + return Optional.of(TYPE_PREVIOUS); + } + + boolean isExternal = contextualExternalVariableApiPort.readContextualExternalFile( + collectionInstrumentId.toUpperCase(), + jsonFilePath.toString() + ); + + if (isExternal) { + return Optional.of(TYPE_EXTERNAL); + } + + return Optional.empty(); + } + + private static void moveFile(String collectionInstrumentId, Mode mode, FileUtils fileUtils, String filePath) throws GenesisException { try { fileUtils.moveFiles(Path.of(filePath), fileUtils.getDoneFolder(collectionInstrumentId, mode.getFolder())); @@ -142,18 +231,4 @@ private VariableQualityToolDto extractValue(Object variable, String variableName ); return variableQualityToolDto; } - - /** - * @return true if any contextual variable part found in file, false otherwise - */ - private boolean processContextualVariableFile(String collectionInstrumentId, Path jsonFilePath) throws GenesisException { - return contextualPreviousVariableApiPort.readContextualPreviousFile( - collectionInstrumentId.toUpperCase(), - null, - jsonFilePath.toString() - ) || contextualExternalVariableApiPort.readContextualExternalFile( - collectionInstrumentId.toUpperCase(), - jsonFilePath.toString() - ); - } } diff --git a/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java b/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java index 34ca174c1..fbc12251b 100644 --- a/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java +++ b/src/test/java/fr/insee/genesis/controller/rest/responses/ContextualVariableControllerTest.java @@ -2,6 +2,8 @@ import fr.insee.genesis.Constants; import fr.insee.genesis.TestConstants; +import fr.insee.genesis.controller.dto.ContextualVariableFileReportDto; +import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto; import fr.insee.genesis.domain.model.contextualvariable.ContextualVariableModel; import fr.insee.genesis.domain.model.surveyunit.Mode; import fr.insee.genesis.domain.ports.api.ContextualExternalVariableApiPort; @@ -18,10 +20,13 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.mockito.junit.jupiter.MockitoSettings; import org.mockito.quality.Strictness; +import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; + import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doReturn; @@ -87,11 +92,41 @@ void saveContextualVariables() { //THEN verify(contextualVariableApiPort, times(1)).saveContextualVariableFiles( eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID), - any(), any() ); } + @Test + @SneakyThrows + void saveContextualVariablesWithReport() { + // GIVEN + SaveContextualVariablesReportDto report = new SaveContextualVariablesReportDto( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, + 1, + List.of(new ContextualVariableFileReportDto("ok.json", "PREVIOUS")) + ); + + doReturn(report).when(contextualVariableApiPort).saveContextualVariableFilesWithReport( + eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID), + any() + ); + + // WHEN + ResponseEntity response = + contextualVariableController.saveContextualVariablesWithReport( + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID + ); + + // THEN + verify(contextualVariableApiPort, times(1)).saveContextualVariableFilesWithReport( + eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID), + any() + ); + + Assertions.assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + Assertions.assertThat(response.getBody()).isEqualTo(report); + } + @Test @SneakyThrows void readContextualPreviousJson() { @@ -134,7 +169,7 @@ void readContextualPreviousJson() { } @Test - void readContextualPreviousJson_notJson() throws GenesisException { + void readContextualPreviousJson_notJson() { Assertions.assertThatThrownBy(() -> contextualVariableController.readContextualPreviousJson( TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, Mode.WEB, @@ -180,13 +215,13 @@ void readContextualExternalJson() { //THEN verify(contextualExternalVariableApiPort).readContextualExternalFile( - eq(TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID.toUpperCase()), - eq(expectedFilePath) + TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID.toUpperCase(), + expectedFilePath ); } @Test - void readContextualExternalJson_notJson() throws GenesisException { + void readContextualExternalJson_notJson() { Assertions.assertThatThrownBy(() -> contextualVariableController.readContextualExternalJson( TestConstants.DEFAULT_COLLECTION_INSTRUMENT_ID, diff --git a/src/test/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonServiceTest.java b/src/test/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonServiceTest.java index 3196b6a75..122e94854 100644 --- a/src/test/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonServiceTest.java +++ b/src/test/java/fr/insee/genesis/domain/service/contextualvariable/ContextualVariableJsonServiceTest.java @@ -1,6 +1,9 @@ package fr.insee.genesis.domain.service.contextualvariable; +import fr.insee.genesis.Constants; import fr.insee.genesis.TestConstants; +import fr.insee.genesis.controller.dto.ContextualVariableFileReportDto; +import fr.insee.genesis.controller.dto.SaveContextualVariablesReportDto; import fr.insee.genesis.controller.dto.VariableQualityToolDto; import fr.insee.genesis.controller.dto.VariableStateDto; import fr.insee.genesis.domain.model.contextualvariable.ContextualExternalVariableModel; @@ -35,6 +38,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -63,8 +67,12 @@ void setUp() throws IOException { FileSystemUtils.deleteRecursively(TEST_FOLDER_PATH); } Files.createDirectories(TEST_FOLDER_PATH); - Files.createFile(TEST_FOLDER_PATH.resolve("ok.json")); - Files.createFile(TEST_FOLDER_PATH.resolve("ok2.json")); + + Path contextualFolderPath = TEST_FOLDER_PATH.resolve("contextual"); + Files.createDirectories(contextualFolderPath); + + Files.createFile(contextualFolderPath.resolve("ok.json")); + Files.createFile(contextualFolderPath.resolve("ok2.json")); } @Test @@ -313,6 +321,8 @@ void saveContextualVariableFiles() { Path.of("ok2.json") ); String doneFolder = "testDone"; + doReturn(TEST_FOLDER_PATH.toString()).when(fileUtils) + .getDataFolder(eq(collectionInstrumentId), eq("WEB"), isNull()); doReturn(doneFolder).when(fileUtils).getDoneFolder(anyString(),anyString()); doReturn(pathList).when(fileUtils).listFiles(anyString()); doReturn(true).when(contextualPreviousVariableApiPort) @@ -326,9 +336,7 @@ void saveContextualVariableFiles() { //WHEN int fileCount = contextualVariableJsonService.saveContextualVariableFiles( collectionInstrumentId, - fileUtils, - TEST_FOLDER_PATH.toString() - ); + fileUtils); //THEN Assertions.assertThat(fileCount).isEqualTo(pathList.size()); @@ -338,6 +346,48 @@ void saveContextualVariableFiles() { ); } + @Test + @SneakyThrows + void saveContextualVariableFilesWithReport() { + // GIVEN + String collectionInstrumentId = "test"; + FileUtils fileUtils = mock(FileUtils.class); + String doneFolder = "testDone"; + doReturn(TEST_FOLDER_PATH.toString()).when(fileUtils) + .getDataFolder(eq(collectionInstrumentId), eq("WEB"), isNull()); + doReturn(doneFolder).when(fileUtils).getDoneFolder(anyString(), anyString()); + + doReturn(true).when(contextualPreviousVariableApiPort) + .readContextualPreviousFile(any(), any(), any()); + + doAnswer(invocation -> + Files.deleteIfExists(invocation.getArgument(0)) + ).when(fileUtils).moveFiles(any(Path.class), any()); + + // WHEN + SaveContextualVariablesReportDto report = + contextualVariableJsonService.saveContextualVariableFilesWithReport( + collectionInstrumentId, + fileUtils + ); + + // THEN + Assertions.assertThat(report).isNotNull(); + Assertions.assertThat(report.questionnaireId()).isEqualTo(collectionInstrumentId); + Assertions.assertThat(report.processedFiles()).isEqualTo(2); + + Assertions.assertThat(report.files()) + .hasSize(2) + .extracting(ContextualVariableFileReportDto::fileName) + .containsExactlyInAnyOrder("ok.json", "ok2.json"); + + Assertions.assertThat(report.files()) + .extracting(ContextualVariableFileReportDto::type) + .containsOnly("PREVIOUS"); + + verify(fileUtils, times(2)).moveFiles(any(Path.class), eq(doneFolder)); + } + @AfterEach void clean() throws IOException { if (Files.exists(TEST_FOLDER_PATH)){