use maxRecordCount 1 to get total features

NekoLyn · NekoLyn · commit 457f20beebee · 2026-03-03T08:44:31.000+11:00
diff --git a/server/src/main/java/au/org/aodn/ogcapi/server/core/service/geoserver/wfs/DownloadWfsDataService.java b/server/src/main/java/au/org/aodn/ogcapi/server/core/service/geoserver/wfs/DownloadWfsDataService.java
@@ -3,22 +3,17 @@
 import au.org.aodn.ogcapi.server.core.configuration.CacheConfig;
 import au.org.aodn.ogcapi.server.core.model.ogc.FeatureRequest;
 import au.org.aodn.ogcapi.server.core.util.DatetimeUtils;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
-import net.opengis.ows10.ExceptionReportType;
-import net.opengis.wfs.FeatureCollectionType;
-import org.geotools.wfs.v1_1.WFSConfiguration;
-import org.geotools.xsd.Parser;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.http.*;
 import org.springframework.web.client.RestTemplate;
 import org.springframework.web.servlet.mvc.method.annotation.SseEmitter;
-import org.xml.sax.SAXException;
 
-import javax.xml.parsers.ParserConfigurationException;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.StringReader;
 import java.math.BigInteger;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -29,7 +24,7 @@ public class DownloadWfsDataService {
     protected final RestTemplate restTemplate;
     protected final HttpEntity<?> pretendUserEntity;
     protected final int chunkSize;
-    protected static final WFSConfiguration WFS_CONFIG = new WFSConfiguration();
+    protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
     protected static final int SAMPLES_SIZE = 500;    // A not too small sample for download size estimation
 
     public DownloadWfsDataService(
@@ -43,6 +38,7 @@ public DownloadWfsDataService(
         this.chunkSize = chunkSize;
         this.restTemplate = restTemplate;
     }
+
     /**
      * Does collection lookup, WFS validation, field retrieval, and URL building
      */
@@ -88,10 +84,12 @@ public String prepareWfsRequestUrl(
             throw new IllegalArgumentException("No WFS server URL found for the given UUID and layer name");
         }
     }
+
     /**
      * We just need to estimate the download size, the way we do it is issue two query:
      * a. Issue a query and get the number or record hit
      * b. Issue a query with data download but then limit the records size, and do a liner interpolation
+     *
      * @return The estimated file size
      */
     @Cacheable(CacheConfig.DOWNLOADABLE_SIZE)
@@ -104,45 +102,40 @@ public BigInteger estimateDownloadSize(
             List<String> fields,
             String outputFormat) throws IllegalArgumentException {
 
-        // Just get number of record, the reply will always in XML
-        String wfsRequestUrl = prepareWfsRequestUrl(
-                uuid, startDate, endDate, multiPolygon, fields, layerName, "",  -1L, true
+        // Get total feature count via GeoJSON response (totalFeatures field is correctly populated
+        // even with spatial CQL filters, unlike resultType=hits which ignores INTERSECTS)
+        String countUrl = prepareWfsRequestUrl(
+                uuid, startDate, endDate, multiPolygon, fields, layerName, "application/json", 1L, false
         );
 
-        ResponseEntity<String> response = restTemplate.exchange(wfsRequestUrl, HttpMethod.GET, pretendUserEntity, String.class);
-
-        if(response.getStatusCode().is2xxSuccessful() && response.getBody() != null) {
-            Parser parser = new Parser(WFS_CONFIG);
-            parser.setValidating(false);
-            parser.setFailOnValidationError(false);
+        ResponseEntity<String> countResponse = restTemplate.exchange(countUrl, HttpMethod.GET, pretendUserEntity, String.class);
 
+        if (countResponse.getStatusCode().is2xxSuccessful() && countResponse.getBody() != null) {
             try {
-                Object o = parser.parse(new StringReader(response.getBody()));
-                if(o instanceof FeatureCollectionType hits) {
-                    BigInteger featureCount = hits.getNumberOfFeatures();
-
-                    log.debug("Total record hits {}", featureCount);
-                    // Now we need to do another query where we limited the record count to something small
-                    wfsRequestUrl = prepareWfsRequestUrl(
-                            uuid, startDate, endDate, multiPolygon, fields, layerName, outputFormat, SAMPLES_SIZE, false
-                    );
-                    ResponseEntity<byte[]> bytes = restTemplate.exchange(wfsRequestUrl, HttpMethod.GET, pretendUserEntity, byte[].class);
-                    if(bytes.getStatusCode().is2xxSuccessful() && bytes.getBody() != null) {
-                        return featureCount
-                                .multiply(BigInteger.valueOf(bytes.getBody().length))
-                                .divide(BigInteger.valueOf(SAMPLES_SIZE));
-                    }
+                JsonNode root = OBJECT_MAPPER.readTree(countResponse.getBody());
+                if (!root.has("totalFeatures")) {
+                    throw new RuntimeException("GeoServer GeoJSON response missing totalFeatures field");
                 }
-                else if(o instanceof ExceptionReportType report) {
-                    throw new IllegalArgumentException(String.join(",", report.getException().stream().map(ex -> ex.getExceptionText().toString()).toList()));
+                BigInteger featureCount = BigInteger.valueOf(root.get("totalFeatures").asLong());
+                log.debug("Total record hits {}", featureCount);
+
+                // Download a small sample to measure bytes per record in the requested output format
+                String sampleUrl = prepareWfsRequestUrl(
+                        uuid, startDate, endDate, multiPolygon, fields, layerName, outputFormat, SAMPLES_SIZE, false
+                );
+                ResponseEntity<byte[]> bytes = restTemplate.exchange(sampleUrl, HttpMethod.GET, pretendUserEntity, byte[].class);
+                if (bytes.getStatusCode().is2xxSuccessful() && bytes.getBody() != null) {
+                    return featureCount
+                            .multiply(BigInteger.valueOf(bytes.getBody().length))
+                            .divide(BigInteger.valueOf(SAMPLES_SIZE));
                 }
-            }
-            catch(IOException | SAXException | ParserConfigurationException e) {
-                log.error("Fail to convert wfs hits result", e);
+            } catch (IOException e) {
+                log.error("Fail to get feature count for estimate", e);
             }
         }
         return null;
     }
+
     /**
      * Execute WFS request with SSE support
      */
diff --git a/server/src/test/java/au/org/aodn/ogcapi/server/core/service/geoserver/wfs/DownloadWfsDataServiceTest.java b/server/src/test/java/au/org/aodn/ogcapi/server/core/service/geoserver/wfs/DownloadWfsDataServiceTest.java
@@ -315,31 +315,17 @@ void shouldReturnEstimatedSizeWhenBothRequestsSucceed() {
         List<String> fields = List.of("name", "area");
         String format = "application/json";
 
-        // 1. Hits response (XML)
-        String hitsXml = """
-            <?xml version="1.0" encoding="UTF-8"?>
-            <wfs:FeatureCollection
-                xmlns:xs="http://www.w3.org/2001/XMLSchema"
-                xmlns:wfs="http://www.opengis.net/wfs"
-                xmlns:gml="http://www.opengis.net/gml"
-                xmlns:ogc="http://www.opengis.net/ogc"
-                xmlns:ows="http://www.opengis.net/ows"
-                xmlns:xlink="http://www.w3.org/1999/xlink"
-                xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-                numberOfFeatures="227193"
-                timeStamp="2026-03-01T22:28:56.206Z"
-                xsi:schemaLocation="http://www.opengis.net/wfs http://schemas.opengis.net/wfs/1.1.0/wfs.xsd"
-            />
-            """;
-        ResponseEntity<String> hitsResponse = new ResponseEntity<>(hitsXml, HttpStatus.OK);
-
-        // 2. Sample response (small payload)
+        // 1. Count response: GeoJSON with totalFeatures (1 record requested, but totalFeatures = full count)
+        String countJson = "{\"totalFeatures\": 227193, \"features\": []}";
+        ResponseEntity<String> countResponse = new ResponseEntity<>(countJson, HttpStatus.OK);
+
+        // 2. Sample response (small payload in requested format)
         byte[] sampleBytes = "fake data".getBytes();
         ResponseEntity<byte[]> sampleResponse = new ResponseEntity<>(sampleBytes, HttpStatus.OK);
 
-        doReturn(hitsResponse)
+        doReturn(countResponse)
                 .when(restTemplate).exchange(
-                    argThat((String url) -> url != null && url.contains("resultType=hits")),
+                    argThat((String url) -> url != null && url.contains("maxFeatures=1")),
                     eq(HttpMethod.GET),
                     any(HttpEntity.class),
                     eq(String.class));
@@ -364,53 +350,48 @@ void shouldReturnEstimatedSizeWhenBothRequestsSucceed() {
         BigInteger size = downloadWfsDataService.estimateDownloadSize(
                 uuid, layer, start, end, multiPolygon, fields, format);
 
-        // Should have call with resultType=hits to get number of record
+        // Should call with maxFeatures=1 to get totalFeatures count via GeoJSON
         verify(restTemplate).exchange(
-                eq("https://dummy.com/wfs?VERSION=1.1.0&typeName=water_bodies&SERVICE=WFS&REQUEST=GetFeature&resultType=hits&propertyName=name,area&cql_filter=((time DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))"),  // or contains(...)
+                argThat((String url) -> url != null && url.contains("maxFeatures=1") && url.contains("outputFormat=application")),
                 eq(HttpMethod.GET),
                 any(),
-                eq(String.class)                    // or byte[].class etc.
+                eq(String.class)
         );
 
-        // Should also call with maxFeatures
+        // Should also call with maxFeatures=500 to sample bytes for size interpolation
         verify(restTemplate).exchange(
-                eq("https://dummy.com/wfs?REQUEST=GetFeature&propertyName=name,area&VERSION=1.0.0&typeName=water_bodies&SERVICE=WFS&outputFormat=application/json&maxFeatures=500&cql_filter=((time DURING 2024-01-01T00:00:00Z/2024-12-31T23:59:59Z))"),  // or contains(...)
+                argThat((String url) -> url != null && url.contains("maxFeatures=" + DownloadWfsDataService.SAMPLES_SIZE)),
                 eq(HttpMethod.GET),
                 any(),
-                eq(byte[].class)                    // or byte[].class etc.
+                eq(byte[].class)
         );
 
-        // numberOfFeatures="227193"
+        // totalFeatures=227193, sampleBytes=9 bytes, SAMPLES_SIZE=500
         long expected = 227193L * sampleBytes.length / DownloadWfsDataService.SAMPLES_SIZE;
         assertEquals(BigInteger.valueOf(expected), size, "Size match");
     }
     /**
-     * Expect illegal exception when param is wrong
+     * Expect RuntimeException when GeoServer returns JSON without the totalFeatures field
+     * (e.g. GeoServer returned an error JSON or unexpected structure)
      */
     @Test
-    void throwExceptionWhenHitsRequestFails() {
+    void throwsExceptionWhenCountResponseMissesTotalFeatures() {
 
         String uuid = "lyr-123";
         String layer = "imos:aatams_sattag_dm_profile_map1";
         String start = "2024-01-01";
         String end = "2024-12-31";
-        Object multiPolygon = new Object(); // or real geometry
+        Object multiPolygon = new Object();
         List<String> fields = List.of("name", "area");
         String format = "application/json";
 
-        // 1. Hits response (XML), indicate error
-        String hitsXml = """
-            <?xml version="1.0" encoding="UTF-8"?>
-            <ows:ExceptionReport xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ows="http://www.opengis.net/ows" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0.0" xsi:schemaLocation="http://www.opengis.net/ows https://geoserver-123.aodn.org.au/geoserver/schemas/ows/1.0.0/owsExceptionReport.xsd">
-                <ows:Exception exceptionCode="InvalidParameterValue" locator="typeName">
-                    <ows:ExceptionText>Feature type imos:aatams_sattag_dm_profile_map1 unknown</ows:ExceptionText>
-                </ows:Exception>
-            </ows:ExceptionReport>
-            """;
-        ResponseEntity<String> hitsResponse = new ResponseEntity<>(hitsXml, HttpStatus.OK);
-        doReturn(hitsResponse)
+        // GeoServer returns JSON but without the totalFeatures field
+        String countJson = "{\"error\": \"Feature type unknown\"}";
+        ResponseEntity<String> countResponse = new ResponseEntity<>(countJson, HttpStatus.OK);
+
+        doReturn(countResponse)
                 .when(restTemplate).exchange(
-                        argThat((String url) -> url != null && url.contains("resultType=hits")),
+                        argThat((String url) -> url != null && url.contains("maxFeatures=1")),
                         eq(HttpMethod.GET),
                         any(HttpEntity.class),
                         eq(String.class));
@@ -427,10 +408,10 @@ void throwExceptionWhenHitsRequestFails() {
         doReturn(fs)
                 .when(wfsServer).getDownloadableFields(eq(uuid), any(WfsServer.WfsFeatureRequest.class));
 
-        assertThrows(IllegalArgumentException.class,
+        assertThrows(RuntimeException.class,
                 () -> downloadWfsDataService.estimateDownloadSize(
-                    uuid, layer, start, end, multiPolygon, fields, format)
-        );
+                        uuid, layer, start, end, multiPolygon, fields, format),
+                "Should throw RuntimeException when totalFeatures is missing from count response");
     }
 
     @Test
@@ -439,23 +420,17 @@ void returnsNullWhenParserThrowsException() {
         String layer = "imos:aatams_sattag_dm_profile_map1";
         String start = "2024-01-01";
         String end = "2024-12-31";
-        Object multiPolygon = new Object(); // or real geometry
+        Object multiPolygon = new Object();
         List<String> fields = List.of("name", "area");
         String format = "application/json";
 
-        // 1. A syntax error XML, should not see this but just incase
-        String hitsXml = """
-            <?xml version="1.0" encoding="UTF-8"?>
-            <ows:ExceptionReport23433 xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:ows="http://www.opengis.net/ows" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" version="1.0.0" xsi:schemaLocation="http://www.opengis.net/ows https://geoserver-123.aodn.org.au/geoserver/schemas/ows/1.0.0/owsExceptionReport.xsd">
-                <ows:Exception exceptionCode="InvalidParameterValue" locator="typeName">
-                    <ows:ExceptionText>Feature type imos:aatams_sattag_dm_profile_map1 unknown</ows:ExceptionText>
-                </ows:Exception>
-            </ows:ExceptionReport>
-            """;
-        ResponseEntity<String> hitsResponse = new ResponseEntity<>(hitsXml, HttpStatus.OK);
-        doReturn(hitsResponse)
+        // GeoServer returns malformed JSON — Jackson will throw an IOException, expect null back
+        String malformedJson = "not-valid-json{{{{";
+        ResponseEntity<String> countResponse = new ResponseEntity<>(malformedJson, HttpStatus.OK);
+
+        doReturn(countResponse)
                 .when(restTemplate).exchange(
-                        argThat((String url) -> url != null && url.contains("resultType=hits")),
+                        argThat((String url) -> url != null && url.contains("maxFeatures=1")),
                         eq(HttpMethod.GET),
                         any(HttpEntity.class),
                         eq(String.class));
@@ -475,6 +450,6 @@ void returnsNullWhenParserThrowsException() {
         BigInteger size = downloadWfsDataService.estimateDownloadSize(
                         uuid, layer, start, end, multiPolygon, fields, format);
 
-        assertNull(size, "Size should be null");
+        assertNull(size, "Size should be null when JSON parsing fails");
     }
 }
