Skip to content

Commit 73e43c2

Browse files
LociStarLociStar
authored
Merge pull request #6 from texttechnologylab/develop
Develop
2 parents 0152b72 + 48d72b5 commit 73e43c2

38 files changed

Lines changed: 4673 additions & 1375 deletions

‎.env.example‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
DB_URL=jdbc:postgresql://postgres:5432/udav
88
DB_USER=postgres
99
DB_PASS=postgres
10+
POSTGRES_DB=udav
1011
DB_SCHEMA=public
1112
DB_DIALECT=POSTGRES
1213
# Batch size for database inserts (default: 5000)
@@ -58,5 +59,4 @@ LLM_API_TOKEN=your-api-token-here
5859
# Java Options
5960
# ============================================
6061
# Adjust memory based on your system and data size
61-
JAVA_OPTS=-Xmx2048m -Xms1024m
62-
62+
JAVA_OPTS="-Xmx10G -Xms1024m"

‎Dockerfile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ USER appuser
3333

3434
# Health check
3535
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
36-
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/actuator/health || exit 1
36+
CMD wget --no-verbose --tries=1 --spider http://localhost:8080/actuator/health/liveness || exit 1
3737

3838
# Expose port
3939
EXPOSE 8080

‎docker-compose.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
services:
2+
postgres:
3+
image: postgres:17-alpine
4+
container_name: udav-postgres
5+
restart: unless-stopped
6+
environment:
7+
POSTGRES_DB: ${POSTGRES_DB:-udav}
8+
POSTGRES_USER: ${DB_USER:-postgres}
9+
POSTGRES_PASSWORD: ${DB_PASS:-postgres}
10+
ports:
11+
- "5432:5432"
12+
volumes:
13+
- postgres_data:/var/lib/postgresql/data
14+
healthcheck:
15+
test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${POSTGRES_DB:-udav}"]
16+
interval: 10s
17+
timeout: 5s
18+
retries: 5
19+
20+
udav:
21+
build:
22+
context: .
23+
dockerfile: Dockerfile
24+
container_name: udav-app
25+
restart: unless-stopped
26+
depends_on:
27+
postgres:
28+
condition: service_healthy
29+
ports:
30+
- "8080:8080"
31+
environment:
32+
DB_URL: jdbc:postgresql://postgres:5432/${POSTGRES_DB:-udav}
33+
DB_USER: ${DB_USER:-postgres}
34+
DB_PASS: ${DB_PASS:-postgres}
35+
DB_SCHEMA: ${DB_SCHEMA:-public}
36+
DB_DIALECT: ${DB_DIALECT:-POSTGRES}
37+
DB_BATCH_SIZE: ${DB_BATCH_SIZE:-5000}
38+
DB_MAX_IDENT: ${DB_MAX_IDENT:-255}
39+
DUUI_IMPORTER: ${DUUI_IMPORTER:-false}
40+
DUUI_IMPORTER_PATH: /app/data/input
41+
DUUI_IMPORTER_FILE_ENDING: ${DUUI_IMPORTER_FILE_ENDING:-.xmi}
42+
DUUI_IMPORTER_WORKERS: ${DUUI_IMPORTER_WORKERS:-4}
43+
DUUI_IMPORTER_CAS_POOL_SIZE: ${DUUI_IMPORTER_CAS_POOL_SIZE:-8}
44+
DUUI_IMPORTER_TYPE_SYSTEM_PATH: ${DUUI_IMPORTER_TYPE_SYSTEM_PATH:-}
45+
APP_INPUT_DIR: ${APP_INPUT_DIR:-/app/data/input}
46+
PIPELINE_IMPORTER: ${PIPELINE_IMPORTER:-true}
47+
PIPELINE_IMPORTER_FOLDER: ${PIPELINE_IMPORTER_FOLDER:-/app/pipelines}
48+
PIPELINE_IMPORTER_REPLACE_IF_DIFFERENT: ${PIPELINE_IMPORTER_REPLACE_IF_DIFFERENT:-false}
49+
SROUCE_BUILDER: ${SROUCE_BUILDER:-false}
50+
LLM_BASE_URL: ${LLM_BASE_URL:-}
51+
LLM_API_TOKEN: ${LLM_API_TOKEN:-}
52+
JAVA_OPTS: ${JAVA_OPTS:--Xmx20G -Xms512m}
53+
healthcheck:
54+
test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/actuator/health/liveness || exit 1"]
55+
interval: 30s
56+
timeout: 10s
57+
retries: 3
58+
start_period: 40s
59+
volumes:
60+
- ${DUUI_IMPORTER_PATH}:/app/data/input:ro
61+
62+
volumes:
63+
postgres_data:
64+

‎src/main/java/org/texttechnologylab/udav/App.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,12 @@
55
import org.springframework.boot.context.properties.EnableConfigurationProperties;
66
import org.texttechnologylab.udav.importer.config.DUUIImporterProps;
77
import org.texttechnologylab.udav.importer.config.DbProps;
8+
import org.texttechnologylab.udav.importer.config.JsonDataImporterProps;
89
import org.texttechnologylab.udav.importer.config.PipelineImporterProps;
910
import org.texttechnologylab.udav.sources.config.SourceBuilderProps;
1011

1112
@SpringBootApplication
12-
@EnableConfigurationProperties({DbProps.class, DUUIImporterProps.class, SourceBuilderProps.class, PipelineImporterProps.class})
13+
@EnableConfigurationProperties({DbProps.class, DUUIImporterProps.class, SourceBuilderProps.class, JsonDataImporterProps.class, PipelineImporterProps.class})
1314
public class App {
1415
public static void main(String[] args) {
1516
SpringApplication.run(App.class, args);

‎src/main/java/org/texttechnologylab/udav/api/Controller/ConvertionController.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
import com.fasterxml.jackson.databind.JsonNode;
1414
import com.fasterxml.jackson.databind.ObjectMapper;
1515
import org.texttechnologylab.udav.widgets.Widget;
16-
import org.texttechnologylab.udav.widgets.tools.SvgToLaTeXConverter;
16+
import org.texttechnologylab.udav.widgets.svgtolatex.SvgToLaTeXConverter;
1717

1818
@RestController
1919
@RequestMapping("/api/convertions")

‎src/main/java/org/texttechnologylab/udav/api/Repositories/UIMATypeRepository.java‎

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
@Repository
1212
public class UIMATypeRepository {
13+
1314
private final DSLContext dsl;
1415

1516
public UIMATypeRepository(DSLContext dsl) {
@@ -18,23 +19,55 @@ public UIMATypeRepository(DSLContext dsl) {
1819

1920
@Transactional(readOnly = true)
2021
public List<UimaTypeRow> list(int page, int size, String q) {
22+
2123
int p = Math.max(0, page);
2224
int s = Math.max(1, size);
2325

2426
var REG = DSL.table("uima_type_registry");
25-
var F_URI = DSL.field("uima_type_uri", String.class).as("uimaTypeUri");
26-
var F_CNT = DSL.field("row_count", Long.class).as("rowCount");
27+
var JSON = DSL.table("json_data");
28+
29+
var F_URI = DSL.field("uima_type_uri", String.class);
30+
var F_SRC = DSL.field("sourcefile_name", String.class);
31+
var F_CNT = DSL.field("row_count", Long.class);
32+
33+
var condRegistry = (q == null || q.isBlank())
34+
? DSL.noCondition()
35+
: F_URI.likeIgnoreCase("%" + q + "%");
2736

28-
var cond = (q == null || q.isBlank())
37+
var condJson = (q == null || q.isBlank())
2938
? DSL.noCondition()
30-
: DSL.field("uima_type_uri", String.class).likeIgnoreCase("%" + q + "%");
39+
: F_SRC.likeIgnoreCase("%" + q + "%");
3140

32-
return dsl.select(F_URI, F_CNT)
41+
var registryQuery = dsl
42+
.select(
43+
F_URI.as("uimaTypeUri"),
44+
F_CNT.as("rowCount")
45+
)
3346
.from(REG)
34-
.where(cond).and(DSL.field(("row_count")).greaterThan(0))
35-
.orderBy(F_CNT.desc().nullsLast()) // numeric sort, NULLS LAST
47+
.where(condRegistry)
48+
.and(F_CNT.greaterThan(0L));
49+
50+
var jsonQuery = dsl
51+
.select(
52+
F_SRC.as("uimaTypeUri"),
53+
DSL.val(-1L).as("rowCount")
54+
)
55+
.from(JSON)
56+
.where(condJson);
57+
58+
var combined = registryQuery
59+
.unionAll(jsonQuery)
60+
.asTable("combined");
61+
62+
var uri = combined.field("uimaTypeUri", String.class);
63+
var count = combined.field("rowCount", Long.class);
64+
65+
return dsl
66+
.select(uri, count)
67+
.from(combined)
68+
.orderBy(count.desc().nullsLast())
3669
.offset(p * s)
3770
.limit(s)
38-
.fetchInto(UimaTypeRow.class); // ✅ maps by aliased field names
71+
.fetchInto(UimaTypeRow.class);
3972
}
40-
}
73+
}

‎src/main/java/org/texttechnologylab/udav/api/service/PipelineService.java‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,15 @@
44
import com.fasterxml.jackson.databind.ObjectMapper;
55
import org.jooq.DSLContext;
66
import org.jooq.impl.DSL;
7+
import org.jooq.impl.SQLDataType;
78
import org.slf4j.Logger;
89
import org.slf4j.LoggerFactory;
10+
import org.springframework.beans.factory.annotation.Value;
911
import org.springframework.stereotype.Service;
1012
import org.springframework.transaction.annotation.Transactional;
1113
import org.springframework.web.server.ResponseStatusException;
1214

15+
import jakarta.annotation.PostConstruct;
1316
import javax.sql.DataSource;
1417
import java.sql.Connection;
1518
import java.sql.SQLException;
@@ -28,6 +31,9 @@ public class PipelineService {
2831
private final DataSource dataSource;
2932
private final ObjectMapper objectMapper;
3033

34+
@Value("${app.db.schema:public}")
35+
private String schema;
36+
3137
Logger LOGGER = LoggerFactory.getLogger(PipelineService.class);
3238

3339
public PipelineService(SourceBuildService sourceBuildService, DataSource dataSource, ObjectMapper objectMapper) {
@@ -36,17 +42,19 @@ public PipelineService(SourceBuildService sourceBuildService, DataSource dataSou
3642
this.objectMapper = objectMapper;
3743
}
3844

39-
// @PostConstruct
40-
// void ensureTable() throws Exception {
41-
// try (Connection c = dataSource.getConnection()) {
42-
// DSLContext dsl = DSL.using(c);
43-
// dsl.createTableIfNotExists(TABLE)
44-
// .column(COL_NAME, SQLDataType.VARCHAR(255).nullable(false))
45-
// .column(COL_JSON, SQLDataType.CLOB.nullable(false)) // switch to JSONB if on Postgres
46-
// .constraints(DSL.constraint("PK_" + TABLE).primaryKey(COL_NAME))
47-
// .execute();
48-
// }
49-
// }
45+
@PostConstruct
46+
void ensureTable() throws Exception {
47+
try (Connection c = dataSource.getConnection()) {
48+
DSLContext dsl = DSL.using(c);
49+
dsl.createSchemaIfNotExists(DSL.name(schema)).execute();
50+
dsl.createTableIfNotExists(DSL.name(schema, TABLE))
51+
.column(DSL.name(COL_ID), SQLDataType.VARCHAR(255).nullable(false))
52+
.column(DSL.name(COL_NAME), SQLDataType.VARCHAR(255).nullable(false))
53+
.column(DSL.name(COL_JSON), SQLDataType.CLOB.nullable(false))
54+
.constraints(DSL.constraint("PK_" + TABLE).primaryKey(DSL.name(COL_ID)))
55+
.execute();
56+
}
57+
}
5058

5159
@Transactional(readOnly = true)
5260
public List<String> listIds(int page, int size, String q) throws Exception {

0 commit comments

Comments
 (0)