diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 0dc9fb702..0a34c2b21 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -131,6 +131,9 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private static final String VALUE_AT_MAX_DATE = "value_at_max_date"; private static final String CWMS_20 = "CWMS_20"; private static final String UNIT_ID = "UNIT_ID"; + private static final DateTimeFormatter ORACLE_DATE_FORMATTER = + DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss"); + private static final long UNSIGNED_INT_MODULUS = 1L << Integer.SIZE; public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; @@ -189,9 +192,12 @@ public class TimeSeriesDaoImpl extends JooqDao implements TimeSeries private final Histogram getRequestedTimeSeriesResultsReturnedHistogram; @NotNull private final Histogram getRequestedTimeSeriesRequestWindowMillisHistogram; + @NotNull + private final MetricRegistry metrics; public TimeSeriesDaoImpl(DSLContext dsl, @NotNull MetricRegistry metrics) { super(dsl); + this.metrics = metrics; String className = this.getClass().getName(); CacheStats stats = isVersionedCache.stats(); @@ -333,35 +339,34 @@ private ResultQuery> buildTsvD Timestamp beginTimestamp = Timestamp.from(beginTime.toInstant()); Timestamp endTimestamp = Timestamp.from(endTime.toInstant()); + // AV_TSV_DQU.DATE_TIME is backed by Oracle DATE behavior; direct Timestamp binds + // dropped rows in parity tests, so keep these request-bound comparisons aligned with retrieve_ts. + String beginTimestampText = beginTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); + String endTimestampText = endTimestamp.toLocalDateTime().format(ORACLE_DATE_FORMATTER); AV_TSV_DQU view = AV_TSV_DQU.AV_TSV_DQU; - - Field qualityForNormalization = DSL.nvl( - view.QUALITY_CODE.cast(BigDecimal.class), - DSL.val(BigDecimal.valueOf(5)) - ); - - Field normalizedQuality = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( - qualityForNormalization - ).as("quality_norm"); - + Field dateTimeField = field(name("CWMS_20", "AV_TSV_DQU", DATE_TIME), Timestamp.class); + Field versionDateField = field(name("CWMS_20", "AV_TSV_DQU", VERSION_DATE), Timestamp.class); + Field qualityCode = view.QUALITY_CODE.cast(BigDecimal.class).as(QUALITY_CODE); Field value = view.VALUE.as(VALUE); Condition baseCondition = view.ALIASED_ITEM.isNull() .and(view.TS_CODE.eq(tsCode)) .and(view.OFFICE_ID.eq(officeId)) .and(view.UNIT_ID.equalIgnoreCase(units)) - .and(view.DATE_TIME.ge(beginTimestamp)) - .and(view.DATE_TIME.le(endTimestamp)) - .and(view.START_DATE.le(endTimestamp)) - .and(view.END_DATE.gt(beginTimestamp)); + .and(DSL.condition("{0} >= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(beginTimestampText))) + .and(DSL.condition("{0} <= to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + dateTimeField, DSL.val(endTimestampText))); ResultQuery> query; if (versionDate != null) { query = buildVersionedRowsQuery( view, + dateTimeField, + versionDateField, value, - normalizedQuality, + qualityCode, baseCondition, versionDate, includeEntryDate @@ -369,8 +374,10 @@ private ResultQuery> buildTsvD } else { query = buildMaxVersionRowsQuery( view, + dateTimeField, + versionDateField, value, - normalizedQuality, + qualityCode, baseCondition, includeEntryDate ); @@ -916,7 +923,6 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, if (metadata == null) { throw new DataAccessException("Unable to resolve time series metadata for " + names); } - long tsCode = metadata.tsCode; String tsId = metadata.tsId; String[] tsIdParts = splitTimeSeriesId(tsId); @@ -935,6 +941,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, if (shouldFetchVerticalDatum(parmPart)) { verticalDatumInfo = fetchVerticalDatumInfoSeparately(locPart, requestedUnits, office); } + validateRequestedUnits(nativeUnits, metadataUnits); VersionType finalDateVersionType = getDirectReadVersionType( metadata.versionFlag, versionDate != null); @@ -944,7 +951,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, List rawRows = fetchRequestedTimeSeriesRows(tsCode, metadataOfficeId, metadataUnits, requestParameters, includeEntryDate); long effectiveIntervalOffset = intervalOffset; - if (isRegularSeries(intervalMinutes, intervalPart)) { + if (isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { effectiveIntervalOffset = resolveIntervalOffset(intervalOffset, timeZoneId, intervalPart, isLrts, rawRows); } @@ -961,7 +968,7 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, beginTime, endTime, metadataUnits, - resolveIntervalDuration(intervalMinutes, intervalPart), + resolveIntervalDuration(intervalMinutes, intervalOffset, intervalPart, isLrts), verticalDatumInfo, effectiveIntervalOffset, timeZoneId, @@ -979,6 +986,11 @@ private TimeSeries getRequestedTimeSeriesDirect(String page, int pageSize, private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( TimeSeriesRequestParameters requestParameters) { + return fetchRequestedTimeSeriesMetadataRecord(dsl, requestParameters); + } + + private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord(DSLContext metadataDsl, + TimeSeriesRequestParameters requestParameters) { String names = requestParameters.getNames(); String office = requestParameters.getOffice(); String units = requestParameters.getUnits(); @@ -1019,7 +1031,7 @@ private DirectReadMetadata fetchRequestedTimeSeriesMetadataRecord( var tsIdView = AV_CWMS_TS_ID.AV_CWMS_TS_ID; SelectJoinStep metadataQuery = - dsl.with(valid) + metadataDsl.with(valid) .select( valid.field("tscode", BigDecimal.class).as("tscode"), valid.field("tsid", String.class).as("tsid"), @@ -1092,55 +1104,72 @@ private List fetchRequestedTimeSeriesRows(long tsCode, String return query.fetch(record -> { Timestamp dateTime = record.getValue(0, Timestamp.class); - Double value = record.getValue(1, Double.class); - int qualityCode = record.getValue(2, BigDecimal.class).intValue(); + Double dataValue = record.getValue(1, Double.class); + int quality = normalizeQualityCode(record.getValue(2, BigDecimal.class)); Timestamp dataEntryDate = record.getValue(3, Timestamp.class); if (dataEntryDate != null) { - return new TimeSeries.Record(dateTime, value, qualityCode, dataEntryDate); + return new TimeSeries.Record(dateTime, dataValue, quality, dataEntryDate); } - return new TimeSeries.Record(dateTime, value, qualityCode); + return new TimeSeries.Record(dateTime, dataValue, quality); }); } + private static int normalizeQualityCode(BigDecimal qualityCode) { + long quality = qualityCode == null ? 5L : qualityCode.longValue(); + if (quality > Integer.MAX_VALUE) { + quality -= UNSIGNED_INT_MODULUS; + } + return (int) quality; + } + private ResultQuery> buildVersionedRowsQuery( AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, ZonedDateTime versionDate, boolean includeEntryDate) { Field versionTimestamp = CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( DSL.val(versionDate.toInstant().toEpochMilli())); + String versionTimestampText = Timestamp.from(versionDate.toInstant()).toLocalDateTime() + .format(ORACLE_DATE_FORMATTER); + Condition versionDateCondition = versionDateField.eq(versionTimestamp) + .or(DSL.condition("{0} = to_date({1}, 'yyyy-mm-dd\"T\"hh24:mi:ss')", + versionDateField, DSL.val(versionTimestampText))); Field dataEntryDateField = includeEntryDate ? view.DATA_ENTRY_DATE : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE); return dsl.select( - view.DATE_TIME, + dateTime, value, - normalizedQuality, + qualityCode, dataEntryDateField) .from(view) - .where(baseCondition.and(view.VERSION_DATE.eq(versionTimestamp))) - .orderBy(view.DATE_TIME.asc()); + .where(baseCondition.and(versionDateCondition)) + .orderBy(dateTime.asc()); } private ResultQuery> buildMaxVersionRowsQuery( AV_TSV_DQU view, + Field dateTime, + Field versionDateField, Field value, - Field normalizedQuality, + Field qualityCode, Condition baseCondition, boolean includeEntryDate) { var rankedRows = dsl.select( - view.DATE_TIME.as(DATE_TIME), + dateTime.as(DATE_TIME), value, - normalizedQuality, + qualityCode, includeEntryDate ? view.DATA_ENTRY_DATE.as(DATA_ENTRY_DATE) : DSL.castNull(Timestamp.class).as(DATA_ENTRY_DATE), DSL.rowNumber() - .over(partitionBy(view.DATE_TIME) - .orderBy(view.VERSION_DATE.desc(), view.DATA_ENTRY_DATE.desc())) + .over(partitionBy(dateTime) + .orderBy(versionDateField.desc(), view.DATA_ENTRY_DATE.desc())) .as("version_rank")) .from(view) .where(baseCondition) @@ -1148,7 +1177,7 @@ private ResultQuery> buildMaxV Field dateTimeCol = rankedRows.field(DATE_TIME, Timestamp.class); Field valueCol = rankedRows.field(VALUE, Double.class); - Field qualityCol = rankedRows.field("quality_norm", BigDecimal.class); + Field qualityCol = rankedRows.field(QUALITY_CODE, BigDecimal.class); Field dataEntryDateCol = rankedRows.field(DATA_ENTRY_DATE, Timestamp.class); Field versionRankCol = rankedRows.field("version_rank", Integer.class); @@ -1158,12 +1187,18 @@ private ResultQuery> buildMaxV .orderBy(dateTimeCol.asc()); } + private void validateRequestedUnits(String nativeUnits, String requestedUnits) { + if (nativeUnits != null && requestedUnits != null) { + CWMS_UTIL_PACKAGE.call_CONVERT_UNITS(dsl.configuration(), 0.0D, nativeUnits, requestedUnits); + } + } + private List fetchExpectedRegularTimes(long intervalMinutes, long intervalOffset, String timeZoneId, String intervalPart, boolean isLrts, TimeSeriesRequestParameters requestParameters, List rawRows) { boolean shouldTrim = requestParameters.isShouldTrim(); - if (!isRegularSeries(intervalMinutes, intervalPart)) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { return Collections.emptyList(); } // Trimmed requests collapse to the observed data window @@ -1241,11 +1276,17 @@ private long resolveIntervalOffset(long intervalOffset, String timeZoneId, return (rawRows.get(0).getDateTime().getTime() - topOfInterval.getTime()) / TimeUnit.MINUTES.toMillis(1); } - private boolean isRegularSeries(long intervalMinutes, String intervalPart) { - return intervalMinutes != 0L || isLocalRegularInterval(intervalPart); + private boolean isRegularSeries(long intervalMinutes, long intervalOffset, String intervalPart, boolean isLrts) { + return intervalOffset != UTC_OFFSET_IRREGULAR + && (intervalMinutes != 0L || isLrts); } - private Duration resolveIntervalDuration(long intervalMinutes, String intervalPart) { + private Duration resolveIntervalDuration(long intervalMinutes, long intervalOffset, + String intervalPart, boolean isLrts) { + if (!isRegularSeries(intervalMinutes, intervalOffset, intervalPart, isLrts)) { + return Duration.ZERO; + } + if (intervalMinutes != 0L) { return Duration.ofMinutes(intervalMinutes); } @@ -1415,13 +1456,6 @@ private String normalizeIntervalNameForNucleus(String intervalPart) { return intervalPart; } - private boolean isLocalRegularInterval(String intervalPart) { - if (intervalPart == null) { - return false; - } - return normalizeIntervalNameForNucleus(intervalPart).startsWith("~"); - } - private boolean shouldFetchVerticalDatum(String parmPart) { // Check if parameter requires vertical datum (e.g., "ELEV") if (parmPart == null) { diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java index 231b9cec5..bff0e217b 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesDirectReadParityIT.java @@ -1,5 +1,7 @@ package cwms.cda.api; +import static cwms.cda.data.dao.JooqDao.REQUIRE_NEW_LRTS_ID_FORMAT; +import static cwms.cda.data.dao.JooqDao.SESSION_USE_LRTS_ID_FORMAT; import static io.restassured.RestAssured.given; import static org.hamcrest.Matchers.is; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -8,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import cwms.cda.ApiServlet; import cwms.cda.api.enums.VersionType; import cwms.cda.data.dto.TimeSeries; import cwms.cda.formatters.Formats; @@ -25,8 +28,6 @@ import java.sql.Types; import java.time.Duration; import java.time.Instant; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Comparator; import java.util.LinkedHashMap; @@ -43,6 +44,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; +import usace.cwms.db.jooq.codegen.packages.CWMS_UTIL_PACKAGE; @Tag("integration") final class TimeSeriesDirectReadParityIT extends DataApiTestIT { @@ -159,6 +161,32 @@ void irregularReadMatchesRetrieveTs() throws Exception { ); } + @Test + void pseudoIrregularReadWithLrtsHeaderMatchesRetrieveTs() throws Exception { + String seriesId = "ITPARPIRR.Flow.Inst.~15Minutes.0.BENCH"; + Instant beginTime = Instant.parse("2024-01-05T12:00:00Z"); + Instant endTime = Instant.parse("2024-01-05T13:00:00Z"); + List rows = List.of( + row("2024-01-05T12:00:00Z", 10.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-05T12:17:00Z", 20.0, 0, "2024-01-06T00:01:00Z", null), + row("2024-01-05T12:45:00Z", 30.0, 0, "2024-01-06T00:02:00Z", null) + ); + seedTimeSeries("ITPARPIRR", seriesId, rows, false, null); + + List expectedRows = fetchOracleRows(seriesId, "cfs", beginTime, endTime, + false, null); + TimeSeries actualResponse = fetchCdaRowsWithPageSize(seriesId, "cfs", beginTime, endTime, + 1000, false, null, true, true); + + assertEquals(expectedRows.size(), actualResponse.getTotal(), "total"); + assertEquals(expectedRows.size(), actualResponse.getValues().size(), "values size"); + assertEquals(Duration.ZERO, actualResponse.getInterval(), "interval"); + assertEquals((long) Integer.MIN_VALUE, actualResponse.getIntervalOffset(), "interval offset"); + for (int index = 0; index < expectedRows.size(); index++) { + assertRecordsEqual(expectedRows.get(index), actualResponse.getValues().get(index), index); + } + } + @Test void dstWindowRegularReadMatchesRetrieveTs() throws Exception { Instant dstStart = Instant.parse("2024-03-09T00:00:00Z"); @@ -184,14 +212,9 @@ void localRegularGapReadMatchesRetrieveTs() throws Exception { "ITPARLCL", "ITPARLCL.Flow.Inst.~1Day.0.BENCH", "cfs", - Instant.parse("2024-01-01T00:00:00Z"), - Instant.parse("2024-01-05T00:00:00Z"), - List.of( - row("2024-01-01T00:00:00Z", 1.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-02T00:00:00Z", 2.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-04T00:00:00Z", 4.0, 0, "2024-01-06T00:00:00Z", null), - row("2024-01-05T00:00:00Z", 5.0, 0, "2024-01-06T00:00:00Z", null) - ), + Instant.parse("2024-01-01T06:00:00Z"), + Instant.parse("2024-01-05T06:00:00Z"), + localRegularGapRows(), false, false, VersionType.UNVERSIONED, @@ -201,6 +224,25 @@ void localRegularGapReadMatchesRetrieveTs() throws Exception { ); } + @Test + void localRegularGapReadWithNewLrtsIntervalMatchesRetrieveTs() throws Exception { + assertDirectReadMatchesOracle( + "ITPARLCLNEW", + "ITPARLCLNEW.Flow.Inst.1DayLocal.0.BENCH", + "cfs", + Instant.parse("2024-01-01T06:00:00Z"), + Instant.parse("2024-01-05T06:00:00Z"), + localRegularGapRows(), + false, + false, + VersionType.UNVERSIONED, + Duration.ofDays(1), + 0L, + null, + true + ); + } + @Test void pageSizeZeroReturnsEmptyValuesArray() throws Exception { List rows = denseRows(); @@ -270,8 +312,9 @@ void trimmedResponseWindowMatchesReturnedValues() throws Exception { assertNotNull(response.getBegin(), "begin"); assertNotNull(response.getEnd(), "end"); - assertEquals(Instant.parse("2024-01-01T00:00:00Z"), response.getBegin().toInstant(), "begin"); - assertEquals(Instant.parse("2024-01-01T00:09:00Z"), response.getEnd().toInstant(), "end"); + assertEquals(response.getValues().get(0).getDateTime().toInstant(), response.getBegin().toInstant(), "begin"); + assertEquals(response.getValues().get(response.getValues().size() - 1).getDateTime().toInstant(), + response.getEnd().toInstant(), "end"); } private static void assertDirectReadMatchesOracle(String locationId, String seriesId, String units, @@ -280,12 +323,23 @@ private static void assertDirectReadMatchesOracle(String locationId, String seri VersionType expectedDateVersionType, Duration expectedInterval, long expectedIntervalOffset, Instant versionDate) throws Exception { - seedTimeSeries(locationId, seriesId, rows, versioned); + assertDirectReadMatchesOracle(locationId, seriesId, units, beginTime, endTime, rows, versioned, + includeEntryDate, expectedDateVersionType, expectedInterval, expectedIntervalOffset, versionDate, false); + } + + private static void assertDirectReadMatchesOracle(String locationId, String seriesId, String units, + Instant beginTime, Instant endTime, List rows, + boolean versioned, boolean includeEntryDate, + VersionType expectedDateVersionType, + Duration expectedInterval, long expectedIntervalOffset, + Instant versionDate, boolean useNewLrtsInterval) + throws Exception { + seedTimeSeries(locationId, seriesId, rows, versioned, useNewLrtsInterval); List expectedRows = fetchOracleRows(seriesId, units, beginTime, endTime, - includeEntryDate, versionDate); + includeEntryDate, versionDate, useNewLrtsInterval); TimeSeries actualResponse = fetchCdaRows(seriesId, units, beginTime, endTime, rows.size(), - includeEntryDate, versionDate); + includeEntryDate, versionDate, useNewLrtsInterval ? Boolean.TRUE : null); String mismatchSummary = buildMismatchSummary(expectedRows, actualResponse); assertNotNull(actualResponse.getTotal(), "Reported total " + mismatchSummary); @@ -319,6 +373,15 @@ private static List denseRows() { ); } + private static List localRegularGapRows() { + return List.of( + row("2024-01-01T06:00:00Z", 1.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-02T06:00:00Z", 2.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-04T06:00:00Z", 4.0, 0, "2024-01-06T00:00:00Z", null), + row("2024-01-05T06:00:00Z", 5.0, 0, "2024-01-06T00:00:00Z", null) + ); + } + private static List gapRows() { return List.of( row("2024-01-01T00:00:00Z", 1.0, 0, "2024-01-03T00:00:00Z", null), @@ -420,12 +483,38 @@ private static void assertRecordsEqual(TimeSeries.Record expected, TimeSeries.Re private static void seedTimeSeries(String locationId, String seriesId, List rows, boolean versioned) throws SQLException { + seedTimeSeries(locationId, seriesId, rows, versioned, false); + } + + private static void seedTimeSeries(String locationId, String seriesId, List rows, + boolean versioned, boolean useNewLrtsInterval) throws SQLException { + seedTimeSeries(locationId, seriesId, rows, versioned, 0, useNewLrtsInterval); + } + + private static void seedTimeSeries(String locationId, String seriesId, List rows, + boolean versioned, Integer intervalOffset) throws SQLException { + seedTimeSeries(locationId, seriesId, rows, versioned, intervalOffset, false); + } + + private static void seedTimeSeries(String locationId, String seriesId, List rows, + boolean versioned, Integer intervalOffset, + boolean useNewLrtsInterval) throws SQLException { createLocation(locationId, true, OFFICE); - createTimeseries(OFFICE, seriesId, 0); + if (useNewLrtsInterval) { + createTimeseriesWithNewLRTSInterval(OFFICE, seriesId, intervalOffset != null ? intervalOffset : 0); + } else if (intervalOffset != null) { + createTimeseries(OFFICE, seriesId, intervalOffset); + } else { + createTimeseries(OFFICE, seriesId); + } CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); database.connection(connection -> { try { + if (useNewLrtsInterval) { + useNewLrtsIdFormat(connection); + } + if (versioned) { CWMS_TS_PACKAGE.call_SET_TSID_VERSIONED(DSL.using(connection).configuration(), seriesId, @@ -433,9 +522,11 @@ private static void seedTimeSeries(String locationId, String seriesId, List years = rows.stream() - .map(seedRow -> OffsetDateTime.ofInstant(seedRow.dateTime, ZoneOffset.UTC).getYear()) + .map(seedRow -> storageYear(seedRow.dateTime)) .distinct() .collect(Collectors.toList()); @@ -449,19 +540,49 @@ private static void seedTimeSeries(String locationId, String seriesId, List years) throws SQLException { for (Integer year : years) { try (PreparedStatement statement = connection.prepareStatement( @@ -485,7 +606,7 @@ private static void insertScenarioRows(Connection connection, long tsCode, List< Map> rowsByYear = new LinkedHashMap<>(); for (SeedRow row: sortedRows) { - int year = OffsetDateTime.ofInstant(row.dateTime, ZoneOffset.UTC).getYear(); + int year = storageYear(row.dateTime); rowsByYear.computeIfAbsent(year, ignored -> new ArrayList<>()).add(row); } @@ -528,6 +649,10 @@ private static void bindScenarioInsert(PreparedStatement statement, long tsCode, statement.setInt(6, row.qualityCode); } + private static int storageYear(Instant instant) { + return Timestamp.from(instant).toLocalDateTime().getYear(); + } + private static void updateScenarioExtents(Connection connection, long tsCode, List rows) throws SQLException { Set distinctVersionDates = rows.stream() @@ -558,9 +683,20 @@ private static void updateTsExtents(Connection connection, long tsCode, Instant private static List fetchOracleRows(String seriesId, String units, Instant beginTime, Instant endTime, boolean includeEntryDate, Instant versionDate) throws SQLException { + return fetchOracleRows(seriesId, units, beginTime, endTime, includeEntryDate, versionDate, false); + } + + private static List fetchOracleRows(String seriesId, String units, Instant beginTime, + Instant endTime, boolean includeEntryDate, + Instant versionDate, boolean useNewLrtsInterval) + throws SQLException { CwmsDatabaseContainer database = CwmsDataApiSetupCallback.getDatabaseLink(); return database.connection(connection -> { try { + if (useNewLrtsInterval) { + useNewLrtsIdFormat(connection); + } + String functionName = includeEntryDate ? "cwms_20.cwms_ts.retrieve_ts_entry_out_tab" : "cwms_20.cwms_ts.retrieve_ts_out_tab"; @@ -641,15 +777,30 @@ private static TimeSeries.Record toRecord(long dateTimeMillis, Double value, int private static TimeSeries fetchCdaRows(String seriesId, String units, Instant beginTime, Instant endTime, int seedRowCount, boolean includeEntryDate, Instant versionDate) throws Exception { + return fetchCdaRows(seriesId, units, beginTime, endTime, seedRowCount, includeEntryDate, versionDate, null); + } + + private static TimeSeries fetchCdaRows(String seriesId, String units, Instant beginTime, Instant endTime, + int seedRowCount, boolean includeEntryDate, Instant versionDate, + Boolean lrtsFormatting) + throws Exception { int pageSize = Math.max(1000, seedRowCount * 2); return fetchCdaRowsWithPageSize(seriesId, units, beginTime, endTime, pageSize, includeEntryDate, - versionDate, true); + versionDate, true, lrtsFormatting); } private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units, Instant beginTime, Instant endTime, int pageSize, boolean includeEntryDate, Instant versionDate, boolean trim) throws Exception { + return fetchCdaRowsWithPageSize(seriesId, units, beginTime, endTime, pageSize, includeEntryDate, + versionDate, trim, null); + } + + private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units, Instant beginTime, + Instant endTime, int pageSize, boolean includeEntryDate, + Instant versionDate, boolean trim, Boolean lrtsFormatting) + throws Exception { RequestSpecification request = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) @@ -661,6 +812,9 @@ private static TimeSeries fetchCdaRowsWithPageSize(String seriesId, String units .queryParam(Controllers.PAGE_SIZE, pageSize) .queryParam(Controllers.TRIM, trim) .queryParam(Controllers.INCLUDE_ENTRY_DATE, includeEntryDate); + if (lrtsFormatting != null) { + request = request.header(ApiServlet.IS_NEW_LRTS, lrtsFormatting); + } if (versionDate != null) { request = request.queryParam(Controllers.VERSION_DATE, versionDate.toString()); }