Skip to content

Commit fcd6c2a

Browse files
NekoLynNekoLyn
authored
Merge pull request #249 from aodn/feature/7080-download-estimator
Feature/7080 download estimator
2 parents 54afdc8 + 91211ed commit fcd6c2a

25 files changed

Lines changed: 571 additions & 265 deletions

File tree

‎pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,16 @@
101101
<artifactId>gt-referencing</artifactId>
102102
<version>${org.geotools.version}</version>
103103
</dependency>
104+
<dependency>
105+
<groupId>org.geotools</groupId>
106+
<artifactId>gt-wfs-ng</artifactId>
107+
<version>${org.geotools.version}</version>
108+
</dependency>
109+
<dependency>
110+
<groupId>org.geotools</groupId>
111+
<artifactId>gt-xml</artifactId>
112+
<version>${org.geotools.version}</version>
113+
</dependency>
104114
<!--
105115
https://mvnrepository.com/artifact/org.geotools/gt-epsg-hsql
106116
it is use to lookup EPSG values

‎server/pom.xml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,14 @@
154154
<groupId>org.geotools</groupId>
155155
<artifactId>gt-epsg-hsql</artifactId>
156156
</dependency>
157+
<dependency>
158+
<groupId>org.geotools</groupId>
159+
<artifactId>gt-wfs-ng</artifactId>
160+
</dependency>
161+
<dependency>
162+
<groupId>org.geotools</groupId>
163+
<artifactId>gt-xml</artifactId>
164+
</dependency>
157165
<dependency>
158166
<groupId>org.locationtech.spatial4j</groupId>
159167
<artifactId>spatial4j</artifactId>
@@ -225,6 +233,11 @@
225233
<version>1.5.0</version>
226234
<scope>test</scope>
227235
</dependency>
236+
<dependency>
237+
<groupId>org.xmlunit</groupId>
238+
<artifactId>xmlunit-core</artifactId>
239+
<scope>test</scope>
240+
</dependency>
228241
</dependencies>
229242
<build>
230243
<plugins>

‎server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/GeoServerConfig.java‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
package au.org.aodn.ogcapi.server.core.configuration;
2+
3+
import au.org.aodn.ogcapi.server.core.service.Search;
4+
import au.org.aodn.ogcapi.server.core.service.geoserver.wfs.DownloadWfsDataService;
5+
import au.org.aodn.ogcapi.server.core.service.geoserver.wfs.WfsDefaultParam;
6+
import au.org.aodn.ogcapi.server.core.service.geoserver.wfs.WfsServer;
7+
import au.org.aodn.ogcapi.server.core.service.geoserver.wms.WmsServer;
8+
import au.org.aodn.ogcapi.server.core.util.RestTemplateUtils;
9+
import jakarta.annotation.Nonnull;
10+
import org.springframework.beans.factory.annotation.Qualifier;
11+
import org.springframework.beans.factory.annotation.Value;
12+
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
13+
import org.springframework.context.annotation.Bean;
14+
import org.springframework.context.annotation.Configuration;
15+
import org.springframework.context.annotation.Lazy;
16+
import org.springframework.http.HttpEntity;
17+
import org.springframework.http.HttpHeaders;
18+
import org.springframework.http.HttpStatusCode;
19+
import org.springframework.http.client.ClientHttpRequestFactory;
20+
import org.springframework.http.client.ClientHttpResponse;
21+
import org.springframework.web.client.RestTemplate;
22+
23+
import java.io.IOException;
24+
import java.io.InputStream;
25+
import java.util.ArrayList;
26+
27+
@Configuration
28+
public class GeoServerConfig {
29+
30+
/**
31+
* Some wfs request contains non standard minetype which is not allow by the default rest template
32+
*/
33+
static class WfsCustomResponseWrapper implements ClientHttpResponse {
34+
private final ClientHttpResponse delegate;
35+
36+
public WfsCustomResponseWrapper(ClientHttpResponse delegate) {
37+
this.delegate = delegate;
38+
}
39+
40+
@Override
41+
@Nonnull
42+
public HttpStatusCode getStatusCode() throws IOException {
43+
return delegate.getStatusCode();
44+
}
45+
46+
@Override
47+
@Nonnull
48+
public String getStatusText() throws IOException {
49+
return delegate.getStatusText();
50+
}
51+
52+
@Override
53+
public void close() {
54+
delegate.close();
55+
}
56+
57+
@Override
58+
@Nonnull
59+
public InputStream getBody() throws IOException {
60+
return delegate.getBody();
61+
}
62+
63+
@Override
64+
@Nonnull
65+
public HttpHeaders getHeaders() {
66+
HttpHeaders headers = new HttpHeaders();
67+
headers.putAll(delegate.getHeaders());
68+
String ct = headers.getFirst(HttpHeaders.CONTENT_TYPE);
69+
if (ct != null && ct.contains("subtype=")) {
70+
headers.set(HttpHeaders.CONTENT_TYPE, "text/xml");
71+
}
72+
return headers;
73+
}
74+
}
75+
76+
@ConditionalOnMissingBean(name = "pretendUserEntity")
77+
@Bean("pretendUserEntity")
78+
public HttpEntity<?> createPretendUserEntity() {
79+
// Some server do not allow program to scrap the content, so we need to pretend to be a client
80+
HttpHeaders headers = new HttpHeaders();
81+
headers.set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36");
82+
83+
return new HttpEntity<>(headers);
84+
}
85+
86+
@Bean
87+
public WfsServer createWfsServer(Search search,
88+
RestTemplate restTemplate,
89+
RestTemplateUtils restTemplateUtils,
90+
@Qualifier("pretendUserEntity") HttpEntity<?> entity,
91+
WfsDefaultParam wfsDefaultParam) {
92+
return new WfsServer(search, restTemplate, restTemplateUtils, entity, wfsDefaultParam);
93+
}
94+
95+
@Bean
96+
public WmsServer createWmsServer(Search search, @Lazy WfsServer wfsServer, @Qualifier("pretendUserEntity") HttpEntity<?> entity) {
97+
return new WmsServer(search, wfsServer, entity);
98+
}
99+
100+
@Bean
101+
@ConditionalOnMissingBean(DownloadWfsDataService.class)
102+
public DownloadWfsDataService createDownloadWfsDataService(WfsServer wfsServer,
103+
RestTemplate restTemplate,
104+
@Qualifier("pretendUserEntity") HttpEntity<?> pretendUserEntity,
105+
@Value("${app.sse.chunkSize:16384}") int chunkSize) {
106+
107+
RestTemplate clone = new RestTemplate(restTemplate.getRequestFactory());
108+
clone.setInterceptors(new ArrayList<>(restTemplate.getInterceptors()));
109+
clone.getInterceptors().add((request, body, execution) -> {
110+
ClientHttpResponse resp = execution.execute(request, body);
111+
return new WfsCustomResponseWrapper(resp);
112+
});
113+
114+
clone.setMessageConverters(new ArrayList<>(restTemplate.getMessageConverters()));
115+
clone.setErrorHandler(restTemplate.getErrorHandler());
116+
117+
return new DownloadWfsDataService(wfsServer, clone, pretendUserEntity, chunkSize);
118+
}
119+
}

‎server/src/main/java/au/org/aodn/ogcapi/server/core/configuration/WfsWmsConfig.java‎

Lines changed: 0 additions & 43 deletions
This file was deleted.

‎server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/DatasetDownloadEnums.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package au.org.aodn.ogcapi.server.core.model.enumeration;
22

3+
import au.org.aodn.ogcapi.processes.model.Execute;
34
import lombok.Getter;
45

6+
import java.util.List;
7+
58
/**
69
* the values are used for aws batch's environment variables
710
*/
@@ -28,6 +31,25 @@ public enum Parameter {
2831
Parameter(String value) {
2932
this.value = value;
3033
}
34+
35+
public String getStringInput(Execute input) {
36+
Object value = input.getInputs().get(getValue());
37+
return value == null ? null : value.toString();
38+
}
39+
40+
public Object getObjectInput(Execute input) {
41+
return input.getInputs().get(getValue());
42+
}
43+
44+
public List<String> getListInput(Execute input) {
45+
Object value = input.getInputs().get(getValue());
46+
if(value instanceof List<?> list) {
47+
return list.stream().map(String::valueOf).toList();
48+
}
49+
else {
50+
return null;
51+
}
52+
}
3153
}
3254

3355
@Getter

‎server/src/main/java/au/org/aodn/ogcapi/server/core/model/enumeration/ProcessIdEnum.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,23 @@
55
@Getter
66
public enum ProcessIdEnum {
77
DOWNLOAD_DATASET("download"),
8-
;
8+
DOWNLOAD_WFS_SSE("downloadWfs"),
9+
DOWNLOAD_WFS_ESTIMATE("estimateWfsDownload"),
10+
UNKNOWN("");
911

1012
private final String value;
1113

1214
ProcessIdEnum(String value) {
1315
this.value = value;
1416
}
17+
18+
public static ProcessIdEnum fromString(String text) {
19+
for (ProcessIdEnum e : values()) {
20+
if (e.value.equalsIgnoreCase(text)) {
21+
return e;
22+
}
23+
}
24+
return UNKNOWN;
25+
}
26+
1527
}

‎server/src/main/java/au/org/aodn/ogcapi/server/core/model/ogc/FeatureRequest.java‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package au.org.aodn.ogcapi.server.core.model.ogc;
22

3+
import com.fasterxml.jackson.annotation.JsonIgnore;
34
import io.swagger.v3.oas.annotations.media.Schema;
45
import lombok.*;
56
import lombok.experimental.SuperBuilder;
@@ -15,19 +16,6 @@
1516
@NoArgsConstructor(force = true, access = AccessLevel.PROTECTED) // Need when using @SuperBuilder
1617
@EqualsAndHashCode
1718
public class FeatureRequest implements Serializable {
18-
// Define a fix name for fields, the geoserver data have all sorts of different name,
19-
// map it here so that we hide the complexity of call from UI.
20-
public enum PropertyName {
21-
wildcard,
22-
time;
23-
24-
public static PropertyName fromString(String input) {
25-
if (input == null || "*".equals(input.trim())) {
26-
return wildcard; // or throw exception / return default
27-
}
28-
return valueOf(input.trim());
29-
}
30-
}
3119

3220
@Getter
3321
public enum GeoServerOutputFormat {
@@ -67,7 +55,7 @@ public String toString() {
6755
}
6856

6957
@Schema(description = "Property to be return")
70-
private List<PropertyName> properties;
58+
private List<String> properties;
7159

7260
@Schema(description = "Only records that have a geometry that intersects the bounding box are selected. The bounding box is provided as four or six numbers, depending on whether the coordinate reference system includes a vertical axis (height or depth): * Lower left corner, coordinate axis 1 * Lower left corner, coordinate axis 2 * Minimum value, coordinate axis 3 (optional) * Upper right corner, coordinate axis 1 * Upper right corner, coordinate axis 2 * Maximum value, coordinate axis 3 (optional) The coordinate reference system of the values is WGS 84 long/lat (http://www.opengis.net/def/crs/OGC/1.3/CRS84) unless a different coordinate reference system is specified in the parameter `bbox-crs`. For WGS 84 longitude/latitude the values are in most cases the sequence of minimum longitude, minimum latitude, maximum longitude and maximum latitude. However, in cases where the box spans the antimeridian the first value (west-most box edge) is larger than the third value (east-most box edge). If the vertical axis is included, the third and the sixth number are the bottom and the top of the 3-dimensional bounding box. If a record has multiple spatial geometry properties, it is the decision of the server whether only a single spatial geometry property is used to determine the extent or all relevant geometries.")
7361
private List<BigDecimal> bbox;
@@ -90,6 +78,9 @@ public String toString() {
9078
@Schema(description = "Y")
9179
private BigDecimal y;
9280

81+
@Schema(description = "GeoJson MultiPolygon")
82+
private Object multiPolygon;
83+
9384
@Schema(description = "Data output format")
9485
private GeoServerOutputFormat outputFormat;
9586

@@ -107,4 +98,22 @@ public String toString() {
10798
public Boolean getEnableGeoServerWhiteList() {
10899
return enableGeoServerWhiteList != null ? enableGeoServerWhiteList : Boolean.TRUE;
109100
}
101+
/**
102+
* date time must be of ogc format xxx/yyy or ../yyy or /yyy or xxx/.. or xxx/ etc
103+
* @return The first part represent start date, null for unspecify.
104+
*/
105+
@JsonIgnore
106+
public String getStartDateTime() {
107+
return (datetime == null || (datetime.startsWith("../") || datetime.startsWith("/"))) ?
108+
null : datetime.split("/")[0];
109+
}
110+
/**
111+
* date time must be of ogc format xxx/yyy or ../yyy or /yyy or xxx/.. or xxx/ etc
112+
* @return The end part represent end date, null for unspecify.
113+
*/
114+
@JsonIgnore
115+
public String getEndDateTime() {
116+
return (datetime == null || (datetime.endsWith("/..") || datetime.endsWith("/"))) ?
117+
null : datetime.split("/")[1];
118+
}
110119
}

‎server/src/main/java/au/org/aodn/ogcapi/server/core/service/ElasticSearch.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -633,7 +633,7 @@ protected static FieldValue toFieldValue(String s) {
633633
// }
634634

635635
@Override
636-
public SearchResult<FeatureGeoJSON> searchFeatureSummary(String collectionId, List<FeatureRequest.PropertyName> properties, String filter) {
636+
public SearchResult<FeatureGeoJSON> searchFeatureSummary(String collectionId, List<String> properties, String filter) {
637637
try {
638638
SearchRequest searchRequest = new SearchRequest.Builder()
639639
.index(dataIndexName)

‎server/src/main/java/au/org/aodn/ogcapi/server/core/service/OGCApiService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public abstract class OGCApiService {
4141

4242
public ResponseEntity<FeatureCollectionGeoJSON> getFeature(String collectionId,
4343
FeatureId fid,
44-
List<FeatureRequest.PropertyName> properties,
44+
List<String> properties,
4545
String filter) throws Exception {
4646
switch(fid) {
4747
case summary -> {

‎server/src/main/java/au/org/aodn/ogcapi/server/core/service/Search.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ public interface Search {
1818
ElasticSearchBase.SearchResult<StacCollectionModel> searchCollections(String id);
1919
ElasticSearchBase.SearchResult<StacCollectionModel> searchCollections(List<String> ids, String sortBy);
2020
ElasticSearchBase.SearchResult<StacCollectionModel> searchAllCollections(String sortBy) throws Exception;
21-
ElasticSearchBase.SearchResult<FeatureGeoJSON>searchFeatureSummary(String collectionId, List<FeatureRequest.PropertyName> properties, String filter) throws Exception;
21+
ElasticSearchBase.SearchResult<FeatureGeoJSON>searchFeatureSummary(String collectionId, List<String> properties, String filter) throws Exception;
2222

2323
ElasticSearchBase.SearchResult<StacCollectionModel> searchByParameters(
2424
List<String> targets,

0 commit comments

Comments
 (0)