1+ package org .texttechnologylab .udav .importer ;
2+
3+ import com .fasterxml .jackson .databind .JsonNode ;
4+ import com .fasterxml .jackson .databind .ObjectMapper ;
5+ import org .jooq .DSLContext ;
6+ import org .jooq .Field ;
7+ import org .jooq .Record ;
8+ import org .jooq .Table ;
9+ import org .jooq .impl .DSL ;
10+ import org .jooq .impl .SQLDataType ;
11+ import org .slf4j .Logger ;
12+ import org .slf4j .LoggerFactory ;
13+ import org .springframework .beans .factory .annotation .Value ;
14+ import org .springframework .boot .ApplicationArguments ;
15+ import org .springframework .boot .ApplicationRunner ;
16+ import org .springframework .boot .autoconfigure .condition .ConditionalOnProperty ;
17+ import org .springframework .core .Ordered ;
18+ import org .springframework .core .annotation .Order ;
19+ import org .springframework .stereotype .Component ;
20+ import org .texttechnologylab .udav .api .service .SourceBuildService ;
21+
22+ import org .json .XML ;
23+
24+ import javax .sql .DataSource ;
25+ import java .nio .charset .StandardCharsets ;
26+ import java .nio .file .Files ;
27+ import java .nio .file .Path ;
28+ import java .nio .file .Paths ;
29+ import java .sql .Connection ;
30+ import java .util .stream .Stream ;
31+
32+ import static org .jooq .impl .DSL .*;
33+
34+ @ Order (1 )
35+ @ Component
36+ @ ConditionalOnProperty (name = "app.json-data-import.enabled" , havingValue = "true" )
37+ public class JsonDataImporter implements ApplicationRunner {
38+
39+ private static final String TABLE = "json_data" ;
40+ private static final String COL_NAME = "sourcefile_name" ;
41+ private static final String COL_JSON = "json" ;
42+ private static final Logger LOGGER = LoggerFactory .getLogger (JsonDataImporter .class );
43+
44+ private final DataSource dataSource ;
45+ private final Path folder ;
46+ private final boolean replaceIfDifferent ;
47+ private final ObjectMapper mapper = new ObjectMapper ();
48+ private final SourceBuildService sourceBuildService ;
49+
50+ @ Value ("${app.db.schema:public}" )
51+ private String schema ;
52+
53+ public JsonDataImporter (
54+ DataSource dataSource ,
55+ SourceBuildService sourceBuildService ,
56+ @ Value ("${app.json-data-import.folder:sourcefilesJSON}" ) String folderPath ,
57+ @ Value ("${app.json-data-import.replace-if-different:false}" ) boolean replaceIfDifferent
58+ ) {
59+ this .dataSource = dataSource ;
60+ this .sourceBuildService = sourceBuildService ;
61+ this .folder = Paths .get (folderPath );
62+ this .replaceIfDifferent = replaceIfDifferent ;
63+ }
64+
65+ @ Override
66+ public void run (ApplicationArguments args ) throws Exception {
67+ if (!Files .exists (folder ) || !Files .isDirectory (folder )) {
68+ LOGGER .warn ("sourcefilesJSON folder does not exist or is not a directory: {}" , folder .toAbsolutePath ());
69+ return ;
70+ }
71+
72+ try (Connection connection = dataSource .getConnection ()) {
73+ DSLContext dsl = DSL .using (connection );
74+
75+ // Ensure schema + table
76+ dsl .createSchemaIfNotExists (DSL .name (schema )).execute ();
77+
78+ Table <Record > T = table (name (schema , TABLE ));
79+ Field <String > F_NAME = field (name (schema , TABLE , COL_NAME ), String .class );
80+ Field <String > F_JSON = field (name (schema , TABLE , COL_JSON ), String .class );
81+
82+ dsl .createTableIfNotExists (T )
83+ .column (F_NAME , SQLDataType .VARCHAR (255 ).nullable (false ))
84+ .column (F_JSON , SQLDataType .CLOB .nullable (false ))
85+ .constraints (constraint ("PK_" + TABLE ).primaryKey (F_NAME ))
86+ .execute ();
87+
88+ LOGGER .info ("Ensured schema and table exist: {}.{}" , schema , TABLE );
89+
90+ try (Stream <Path > files = Files .list (folder )) {
91+ files .filter (p -> {
92+ String name = p .getFileName ().toString ().toLowerCase ();
93+ return Files .isRegularFile (p )
94+ && (name .endsWith (".json" ) || name .endsWith (".xml" ));
95+ })
96+ .forEach (p -> importOne (dsl , T , F_NAME , F_JSON , p ));
97+ }
98+ }
99+ }
100+
101+ private void importOne (DSLContext dsl ,
102+ Table <Record > T ,
103+ Field <String > F_NAME ,
104+ Field <String > F_JSON ,
105+ Path p ) {
106+ try {
107+ String raw = Files .readString (p , StandardCharsets .UTF_8 );
108+ String sourceFileName = p .getFileName ().toString ();
109+ String canonicalJson ;
110+
111+ if (sourceFileName .toLowerCase ().endsWith (".xml" )) {
112+ canonicalJson = convertXmlToJson (raw );
113+ } else {
114+ canonicalJson = canonicalize (raw );
115+ }
116+
117+ boolean nameExists = sourceFileNameExists (dsl , T , F_NAME , sourceFileName );
118+
119+ if (!nameExists ) {
120+ dsl .insertInto (T )
121+ .columns (F_NAME , F_JSON )
122+ .values (sourceFileName , canonicalJson )
123+ .execute ();
124+
125+ LOGGER .info ("JSON data with name {} has been inserted." , sourceFileName );
126+ return ;
127+ }
128+
129+ if (replaceIfDifferent ) {
130+ String existingJson = dsl .select (F_JSON ).from (T ).where (F_NAME .eq (sourceFileName )).fetchOne (F_JSON );
131+
132+ String existingCanon = (existingJson == null ) ? null : canonicalize (existingJson );
133+ String newCanon = canonicalize (canonicalJson );
134+
135+ if (existingCanon != null && existingCanon .equals (newCanon )) {
136+ LOGGER .warn ("Skipped {} (unchanged)" , sourceFileName );
137+ return ;
138+ }
139+
140+ int updated = dsl .update (T )
141+ .set (F_JSON , canonicalJson )
142+ .where (F_NAME .eq (sourceFileName ))
143+ .execute ();
144+
145+ LOGGER .info ("JSON data with name {} has been {}." , sourceFileName ,
146+ updated == 1 ? "updated" : "not updated" );
147+ return ;
148+ }
149+
150+ LOGGER .warn ("JSON data with name {} already exists. Skipping." , sourceFileName );
151+
152+ } catch (Exception e ) {
153+ LOGGER .error ("Failed to import JSON data from file {}: {}" , p .getFileName (), e .getMessage ());
154+ }
155+ }
156+
157+ // --- Helpers ---
158+
159+ private boolean sourceFileNameExists (DSLContext dsl , Table <Record > T , Field <String > F_NAME , String name ) {
160+ return dsl .fetchExists (selectOne ().from (T ).where (F_NAME .eq (name )));
161+ }
162+
163+ private String canonicalize (String json ) throws Exception {
164+ JsonNode node = mapper .readTree (json );
165+ return mapper .writeValueAsString (node );
166+ }
167+
168+ private String convertXmlToJson (String xml ) {
169+ return XML .toJSONObject (xml ).toString ();
170+ }
171+ }
0 commit comments