diff --git a/api/src/main/java/org/apache/unomi/api/conditions/Condition.java b/api/src/main/java/org/apache/unomi/api/conditions/Condition.java index a319dddf1e..526ff463f7 100644 --- a/api/src/main/java/org/apache/unomi/api/conditions/Condition.java +++ b/api/src/main/java/org/apache/unomi/api/conditions/Condition.java @@ -31,7 +31,7 @@ public class Condition implements Serializable { ConditionType conditionType; String conditionTypeId; - Map parameterValues = new HashMap(); + Map parameterValues = new HashMap<>(); /** * Instantiates a new Condition. diff --git a/api/src/main/java/org/apache/unomi/api/services/EventService.java b/api/src/main/java/org/apache/unomi/api/services/EventService.java index 7c59b37fc0..fb6d60d70b 100644 --- a/api/src/main/java/org/apache/unomi/api/services/EventService.java +++ b/api/src/main/java/org/apache/unomi/api/services/EventService.java @@ -78,14 +78,6 @@ public interface EventService { */ String authenticateThirdPartyServer(String key, String ip); - /** - * Retrieves the list of available event properties. - * - * @return a list of available event properties - * @deprecated use event types instead - */ - List getEventProperties(); - /** * Retrieves the set of known event type identifiers. * diff --git a/bom/pom.xml b/bom/pom.xml index 85de256f75..7b08f0c2c1 100644 --- a/bom/pom.xml +++ b/bom/pom.xml @@ -144,18 +144,18 @@ - org.elasticsearch.client - elasticsearch-rest-high-level-client + org.elasticsearch.test + framework ${elasticsearch.version} - org.elasticsearch.test - framework + co.elastic.clients + elasticsearch-java ${elasticsearch.version} - org.elasticsearch.plugin - transport-netty4-client + org.elasticsearch.client + elasticsearch-rest-client ${elasticsearch.version} @@ -235,7 +235,6 @@ jackson-jaxrs-json-provider ${jackson.version} - org.apache.lucene lucene-test-framework @@ -297,11 +296,6 @@ opencsv ${opencsv.version} - - com.hazelcast - hazelcast-all - ${hazelcast.version} - net.jodah failsafe @@ -337,11 +331,6 @@ jackson-coreutils ${jackson-coreutils.version} - - com.google.guava - guava - ${guava.version} - org.json json diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/ScriptMetadata.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/ScriptMetadata.java index 4756d97cd8..f8f3803b0d 100644 --- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/ScriptMetadata.java +++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/ScriptMetadata.java @@ -17,6 +17,7 @@ package org.apache.unomi.groovy.actions; import groovy.lang.Script; + import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.nio.charset.StandardCharsets; @@ -44,7 +45,7 @@ public final class ScriptMetadata { /** * Constructs a new ScriptMetadata instance. * - * @param actionName the unique name/identifier of the action + * @param actionName the unique name/identifier of the action * @param scriptContent the raw Groovy script content * @param compiledClass the compiled Groovy script class * @throws IllegalArgumentException if any parameter is null @@ -143,10 +144,9 @@ public long getCreationTime() { * This class can be used to create new script instances for execution * without requiring recompilation. * - * * @return the compiled script class, never null */ public Class getCompiledClass() { return compiledClass; } -} +} \ No newline at end of file diff --git a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/GroovyActionsService.java b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/GroovyActionsService.java index ed2c2f2130..4f74528bbe 100644 --- a/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/GroovyActionsService.java +++ b/extensions/groovy-actions/services/src/main/java/org/apache/unomi/groovy/actions/services/GroovyActionsService.java @@ -38,7 +38,6 @@ *

* Thread Safety: Implementations must be thread-safe as this service * is accessed concurrently during script execution. - * * @see GroovyAction * @see ScriptMetadata * @since 2.7.0 @@ -51,7 +50,6 @@ public interface GroovyActionsService { * This method compiles the script, validates it has the required * annotations, persists it, and updates the internal cache. * If the script content hasn't changed, recompilation is skipped. - * * @param actionName the unique identifier for the action * @param groovyScript the Groovy script source code * @throws IllegalArgumentException if actionName or groovyScript is null @@ -64,7 +62,6 @@ public interface GroovyActionsService { *

* This method removes the action from both the cache and persistent storage, * and cleans up any registered action types in the definitions service. - * * @param actionName the unique identifier of the action to remove * @throws IllegalArgumentException if id is null */ @@ -76,7 +73,6 @@ public interface GroovyActionsService { * This is the preferred method for script execution as it returns * pre-compiled classes without any compilation overhead. Returns * {@code null} if the script is not found in the cache. - * * @param actionName the unique identifier of the action * @return the compiled script class, or {@code null} if not found in cache * @throws IllegalArgumentException if id is null @@ -89,7 +85,6 @@ public interface GroovyActionsService { * The returned metadata includes content hash, compilation timestamp, * and the compiled class reference. This is useful for monitoring * tools and debugging. - * * @param actionName the unique identifier of the action * @return the script metadata, or {@code null} if not found * @throws IllegalArgumentException if actionName is null diff --git a/extensions/healthcheck/src/main/java/org/apache/unomi/healthcheck/servlet/HealthCheckServlet.java b/extensions/healthcheck/src/main/java/org/apache/unomi/healthcheck/servlet/HealthCheckServlet.java index e687b6c374..0156d7c0de 100644 --- a/extensions/healthcheck/src/main/java/org/apache/unomi/healthcheck/servlet/HealthCheckServlet.java +++ b/extensions/healthcheck/src/main/java/org/apache/unomi/healthcheck/servlet/HealthCheckServlet.java @@ -70,10 +70,10 @@ protected void service(HttpServletRequest request, HttpServletResponse response) response.getWriter().println(mapper.writeValueAsString(checks)); response.setContentType("application/json"); response.setHeader("Cache-Control", "no-cache"); - if (checks.stream().allMatch(HealthCheckResponse::isLive)) { - response.setStatus(HttpServletResponse.SC_OK); - } else { - response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); - } + if (checks.stream().allMatch(HealthCheckResponse::isLive)) { + response.setStatus(HttpServletResponse.SC_OK); + } else { + response.setStatus(HttpServletResponse.SC_PARTIAL_CONTENT); + } } } diff --git a/itests/README.md b/itests/README.md index b498318894..5ca328425f 100644 --- a/itests/README.md +++ b/itests/README.md @@ -82,7 +82,7 @@ https://maven.apache.org/surefire/maven-failsafe-plugin/examples/single-test.htm Here's an example: - mvn clean install -Dit.karaf.debug=hold:true -Dit.test=org.apache.unomi.itests.graphql.GraphQLEventIT + mvn clean install -Dit.karaf.debug=hold:true -Dit.test=org.apache.unomi.itests.BasicIT ## Migration tests @@ -137,12 +137,12 @@ public class Migrate16xTo200IT extends BaseIT { // Create snapshot repo HttpUtils.executePutRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/", resourceAsString("migration/create_snapshots_repository.json"), null); // Get snapshot, insure it exists - String snapshot = HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x", null); - if (snapshot == null || !snapshot.contains("snapshot_1.6.x")) { + String snapshot = HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_2", null); + if (snapshot == null || !snapshot.contains("snapshot_2")) { throw new RuntimeException("Unable to retrieve 1.6.x snapshot for ES restore"); } // Restore the snapshot - HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true", "{}", null); + HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_2/_restore?wait_for_completion=true", "{}", null); } catch (IOException e) { throw new RuntimeException(e); } @@ -167,7 +167,7 @@ public class Migrate16xTo200IT extends BaseIT { ### How to update a migration test ElasticSearch Snapshot ? -In the following example we want to modify the snapshot: `snapshot_1.6.x`. +In the following example we want to modify the snapshot: `snapshot_2`. This snapshot has been done on Unomi 1.6.x using ElasticSearch 7.11.0. So we will set up locally those servers in the exact same versions. (For now just download them and do not start them yet.) @@ -217,13 +217,13 @@ Now we have to add the snapshot repository, do the following request on your Ela } Now we need to restore the snapshot we want to modify, -but first let's try to see if the snapshot with the id `snapshot_1.6.x` correctly exists: +but first let's try to see if the snapshot with the id `snapshot_2` correctly exists: - GET /_snapshot/snapshots_repository/snapshot_1.6.x + GET /_snapshot/snapshots_repository/snapshot_2 If the snapshot exists we can restore it: - POST /_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true + POST /_snapshot/snapshots_repository/snapshot_2/_restore?wait_for_completion=true {} At the end of the previous request ElasticSearch should be ready and our Unomi snapshot is restored to version `1.6.x`. @@ -241,11 +241,11 @@ they are probably used by the actual migration tests already.) Once you data updated we need to recreate the snapshot, first we delete the old snapshot: - DELETE /_snapshot/snapshots_repository/snapshot_1.6.x + DELETE /_snapshot/snapshots_repository/snapshot_2 Then we recreate it: - PUT /_snapshot/snapshots_repository/snapshot_1.6.x + PUT /_snapshot/snapshots_repository/snapshot_2 Once the process finished (check the ElasticSearch logs to see that the snapshot is correctly created), we need to remove the snapshot repository from our local ElasticSearch diff --git a/itests/pom.xml b/itests/pom.xml index 6f6e71089f..6d9d1f20de 100644 --- a/itests/pom.xml +++ b/itests/pom.xml @@ -207,8 +207,10 @@ com.github.alexcojocaru elasticsearch-maven-plugin - 6.23 + 6.29 + contextElasticSearchITests 9500 9400 @@ -223,7 +225,6 @@ false ${project.build.directory}/snapshots_repository false - * OPTIONS,HEAD,GET,POST,PUT,DELETE Authorization,X-Requested-With,X-Auth-Token,Content-Type,Content-Length diff --git a/itests/src/test/java/org/apache/unomi/itests/ContextServletIT.java b/itests/src/test/java/org/apache/unomi/itests/ContextServletIT.java index 9e6a85b3c4..7014ec66e5 100644 --- a/itests/src/test/java/org/apache/unomi/itests/ContextServletIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/ContextServletIT.java @@ -843,10 +843,13 @@ public void testConcealedProperties() throws Exception { HttpPost request = new HttpPost(getFullUrl(CONTEXT_URL)); request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.APPLICATION_JSON)); assertEquals(TestUtils.executeContextJSONRequest(request, sessionId).getContextResponse().getProfileProperties().get("customProperty"), ("concealedValue")); - // set the property as concealed + // set the property as concealed customPropertyType.getMetadata().getSystemTags().add("concealed"); profileService.deletePropertyType(customPropertyType.getItemId()); + persistenceService.refreshIndex(PropertyType.class); + Thread.sleep(2000); profileService.setPropertyType(customPropertyType); + // Not in all properties request.setEntity(new StringEntity(objectMapper.writeValueAsString(contextRequest), ContentType.APPLICATION_JSON)); assertNull(TestUtils.executeContextJSONRequest(request, sessionId).getContextResponse().getProfileProperties().get("customProperty")); diff --git a/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java b/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java index e6fd540549..133411e77d 100644 --- a/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/SegmentIT.java @@ -462,7 +462,7 @@ public void testScoringPastEventRecalculationMaximumEventCount() throws Exceptio Condition pastEventCondition = new Condition(definitionsService.getConditionType("pastEventCondition")); pastEventCondition.setParameter("numberOfDays", 10); Condition pastEventEventCondition = new Condition(definitionsService.getConditionType("eventTypeCondition")); - pastEventEventCondition.setParameter("eventTypeId", "test-event-type-max"); + pastEventEventCondition.setParameter("eventTypeId", "testeventtypemax"); pastEventCondition.setParameter("eventCondition", pastEventEventCondition); pastEventCondition.setParameter("maximumEventCount", 1); @@ -481,7 +481,7 @@ public void testScoringPastEventRecalculationMaximumEventCount() throws Exceptio // Persist the event (do not send it into the system so that it will not be processed by the rules) ZoneId defaultZoneId = ZoneId.systemDefault(); LocalDate localDate = LocalDate.now().minusDays(3); - Event testEvent = new Event("test-event-type-max", null, profile, null, null, profile, + Event testEvent = new Event("testeventtypemax", null, profile, null, null, profile, Date.from(localDate.atStartOfDay(defaultZoneId).toInstant())); testEvent.setPersistent(true); persistenceService.save(testEvent, null, true); @@ -503,7 +503,7 @@ public void testScoringPastEventRecalculationMaximumEventCount() throws Exceptio // Persist the 2 event (do not send it into the system so that it will not be processed by the rules) defaultZoneId = ZoneId.systemDefault(); localDate = LocalDate.now().minusDays(3); - testEvent = new Event("test-event-type-max", null, profile, null, null, profile, + testEvent = new Event("testeventtypemax", null, profile, null, null, profile, Date.from(localDate.atStartOfDay(defaultZoneId).toInstant())); testEvent.setPersistent(true); persistenceService.save(testEvent, null, true); diff --git a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java index d8c9494ea6..126674a891 100644 --- a/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java +++ b/itests/src/test/java/org/apache/unomi/itests/migration/Migrate16xToCurrentVersionIT.java @@ -51,12 +51,12 @@ public void waitForStartup() throws InterruptedException { // Create snapshot repo HttpUtils.executePutRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/", resourceAsString("migration/create_snapshots_repository.json"), null); // Get snapshot, insure it exists - String snapshot = HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x", null); - if (snapshot == null || !snapshot.contains("snapshot_1.6.x")) { + String snapshot = HttpUtils.executeGetRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_2", null); + if (snapshot == null || !snapshot.contains("snapshot_2")) { throw new RuntimeException("Unable to retrieve 1.6.x snapshot for ES restore"); } // Restore the snapshot - HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_1.6.x/_restore?wait_for_completion=true", "{}", null); + HttpUtils.executePostRequest(httpClient, "http://localhost:9400/_snapshot/snapshots_repository/snapshot_2/_restore?wait_for_completion=true", "{}", null); // Get initial counts of items to compare after migration initCounts(httpClient); diff --git a/itests/src/test/resources/migration/snapshots_repository.zip b/itests/src/test/resources/migration/snapshots_repository.zip index 010569bc0b..d8af55cd4c 100644 Binary files a/itests/src/test/resources/migration/snapshots_repository.zip and b/itests/src/test/resources/migration/snapshots_repository.zip differ diff --git a/kar/pom.xml b/kar/pom.xml index bc5bca789e..7742f126e8 100644 --- a/kar/pom.xml +++ b/kar/pom.xml @@ -152,7 +152,6 @@ httpclient-osgi - diff --git a/kar/src/main/feature/feature.xml b/kar/src/main/feature/feature.xml index 4c64d920f8..16cf2e8b82 100644 --- a/kar/src/main/feature/feature.xml +++ b/kar/src/main/feature/feature.xml @@ -82,7 +82,9 @@ mvn:org.apache.unomi/unomi-scripting/${project.version} mvn:org.apache.unomi/unomi-metrics/${project.version} mvn:org.apache.unomi/unomi-persistence-spi/${project.version} + mvn:org.apache.unomi/unomi-persistence-elasticsearch-core/${project.version} + mvn:org.apache.unomi/unomi-persistence-elasticsearch-conditions/${project.version} mvn:org.apache.unomi/unomi-services/${project.version} mvn:org.apache.unomi/unomi-json-schema-services/${project.version} mvn:org.apache.unomi/unomi-json-schema-rest/${project.version} diff --git a/lifecycle-watcher/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/lifecycle-watcher/src/main/resources/OSGI-INF/blueprint/blueprint.xml index c7eb443c60..5272b74e58 100644 --- a/lifecycle-watcher/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/lifecycle-watcher/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -45,6 +45,7 @@ + diff --git a/manual/src/main/asciidoc/whats-new.adoc b/manual/src/main/asciidoc/whats-new.adoc index 7efb0e4ff0..d1dce150d9 100644 --- a/manual/src/main/asciidoc/whats-new.adoc +++ b/manual/src/main/asciidoc/whats-new.adoc @@ -11,167 +11,19 @@ // See the License for the specific language governing permissions and // limitations under the License. // -=== What's new in Apache Unomi 2.0 +=== What's new in Apache Unomi 3.0 -Apache Unomi 2 is a new release focused on improving core functionalities and robustness of the product. +Apache Unomi 3 is a new release focused on integrations of the client to support elasticsearch version 9. +It also include the upgrade of the Karaf version. -The introduction of tighter data validation with JSON Schemas required some changes in the product data model, which presented an opportunity for noticeable improvements in the overall performance. +==== Elasticsearch client upgrade -This new release also introduces a first (beta) version of the Unomi GraphQL API. +The official client for Elasticsearch has been added to Apache Unomi in version 3.0 in order to replace the old rest-client which +is not supported anymore. -==== Introducing profiles aliases +The documentation of the client can be found here: https://www.elastic.co/docs/reference/elasticsearch/clients/java -Profiles may now have alias IDs, which is a new way to reference profiles using multiple IDs. The Unomi ID still exists, but a new index with aliases can reference a single Unomi profile. This enables more flexible integrations with external systems, as well as provide more flexible and reliable merging mechanisms. A new REST API makes it easy to define, update and remove aliases for profiles. You can read more about <>. +==== Karaf upgrade -==== Scopes declarations are now required - -Scopes declarations are now required in Unomi 2. When submitting an event and specifying a scope, -that scope must already be declared on the platform. - -Scopes can be easily created via the corresponding REST API (`cxs/scopes`) - -For example, an "apache" scope can be created using the following API call. -[source] ----- -curl --location --request POST 'http://localhost:8181/cxs/scopes' \ --u 'karaf:karaf' \ ---header 'Content-Type: application/json' \ ---data-raw '{ -"itemId": "apache", -"itemType": "scope" -}' ----- - -==== JSON Schemas - -Apache Unomi 2 introduces support for https://json-schema.org/specification.html[JSON Schema] for all of its publicly exposed endpoints. -Data received by Apache Unomi 2 will first be validated against a known schema to make sure it complies with an expected payload. -If the received payload does not match a known schema, it will be rejected by Apache Unomi 2. - -Apache Unomi 2 also introduces a set of administrative endpoints allowing new schemas and/or schemas extensions to be registered. - -More details about JSON Schemas implementation are available in the <> of the documentation. - -==== Updated data model - -The introduction of JSON schema required us to modify Apache Unomi data model, One of the key differences is the removal of open maps. - -The properties field in the events objects provided by unomi are now restricted by JSON schema. -This means object properties must be declared in a JSON schema for an event to be accepted. - -A new property, flattenedProperties has been introduced to the event object, this property has been added to store the properties as -flattened in Elasticsearch and should avoid mapping explosion for dynamic properties. - -If there is dynamic properties that you want to send with your event, you should use the flattenedProperties field of the event. - -It's also necessary to specify the format of the values which are added to flattenedProperties by JSON schema but these value will be -stored as flattened and will not create dynamic mapping contrary to the properties field of the events. - -Here is an example for objects that used dynamic properties for URL parameters: - -The following event example in Apache Unomi 1.x: -[source] ----- -{ - "eventType":"view", - "scope":"digitall", - "properties":{ - "URLParameters":{ - "utm_source":"source" - } - }, - "target":{ - "scope":"digitall", - "itemId":"30c0a9e3-4330-417d-9c66-4c1beec85f08", - "itemType":"page", - "properties":{ - "pageInfo":{ - "pageID":"30c0a9e3-4330-417d-9c66-4c1beec85f08", - "nodeType":"jnt:page", - "pageName":"Home", - ... - }, - "attributes":{}, - "consentTypes":[] - } - }, - "source":{ - "scope":"digitall", - "itemId":"ff5886e0-d75a-4061-9de9-d90dfc9e18d8", - "itemType":"site" - } -} ----- - -Is replaced by the following in Apache Unomi 2.x: -[source] ----- -{ - "eventType":"view", - "scope":"digitall", - "flattenedProperties":{ - "URLParameters":{ - "utm_source":"source" - } - }, - "target":{ - "scope":"digitall", - "itemId":"30c0a9e3-4330-417d-9c66-4c1beec85f08", - "itemType":"page", - "properties":{ - "pageInfo":{ - "pageID":"30c0a9e3-4330-417d-9c66-4c1beec85f08", - "nodeType":"jnt:page", - "pageName":"Home", - ... - }, - "attributes":{}, - "consentTypes":[] - } - }, - "source":{ - "scope":"digitall", - "itemId":"ff5886e0-d75a-4061-9de9-d90dfc9e18d8", - "itemType":"site" - } -} ----- - -If using the default Apache 1.x data model, our Unomi 2 migration process will handle the data model changes for you. - -If you are using custom events/objects, please refer to the detailed <<_migration_overview,migration guide>> for more details. - -==== New Web Tracker - -Apache Unomi 2.0 Web Tracker, located in `extensions/web-tracker/` has been completely rewritten. It is no longer based on an external library and is fully self-sufficient. It is based on an external contribution that has been used in production on many sites. - -You can find more information about the <<_unomi_web_tracking_tutorial,new web tracker here>>. - -==== GraphQL API - beta - -Apache Unomi 2.0 sees the introduction of a new (beta) GraphQL API. -Available behind a feature flag (the API disabled by default), the GraphQL API is available for you to play with. - -More details about how to enable/disable the GraphQL API are available in the <> of the documentation. - -We welcome tickets/PRs to improve its robustness and progressively make it ready for prime time. - -==== Migrate from Unomi 1.x - -To facilitate migration we prepared a set of scripts that will automatically handle the migration of your data from Apache Unomi 1.5+ to Apache Unomi 2.0. - -It is worth keeping in mind that for Apache Unomi 2.0 we do not support “hot” migration, -the migration process will require a shutdown of your cluster to guarantee that no new events will be collected while data migration is in progress. - -Special caution must be taken if you declared custom events as our migration scripts can only handle objects we know of. -More details about migration (incl. of custom events) is available in the corresponding section <> of the documentation. - -==== Elasticsearch compatibility - -We currently recommend using Elasticsearch 7.17.5 with Apache Unomi 2.0, -this ensure you are on a recent version that is not impacted by the log4j vulnerabilities (fixed in Elasticsearch 7.16.3). - -This version increase is releated to Apache Unomi 2.0 makeing use of a new Elasticsearch field type -called https://www.elastic.co/guide/en/elasticsearch/reference/7.17/flattened.html[Flattened], -and although it was available in prior versions of Elasticsearch, we do not recommend using those -due to the above-mentioned log4j vulnerabilities. +The Karaf version has been upgraded from 4.2.15 to 4.4.8 in order to support the latest versions of the dependencies. +This upgrade also brings support for Java 17. diff --git a/metrics/pom.xml b/metrics/pom.xml index aed1b61b95..c84b45d614 100644 --- a/metrics/pom.xml +++ b/metrics/pom.xml @@ -62,12 +62,6 @@ unomi-common provided - - org.apache.unomi - unomi-persistence-spi - provided - - com.fasterxml.jackson.core jackson-core diff --git a/metrics/src/main/java/org/apache/unomi/metrics/commands/ViewCommand.java b/metrics/src/main/java/org/apache/unomi/metrics/commands/ViewCommand.java index b3cefe84c1..0e1ba72727 100644 --- a/metrics/src/main/java/org/apache/unomi/metrics/commands/ViewCommand.java +++ b/metrics/src/main/java/org/apache/unomi/metrics/commands/ViewCommand.java @@ -18,15 +18,16 @@ import com.fasterxml.jackson.core.util.DefaultIndenter; import com.fasterxml.jackson.core.util.DefaultPrettyPrinter; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.karaf.shell.commands.Argument; import org.apache.karaf.shell.commands.Command; import org.apache.unomi.metrics.Metric; -import org.apache.unomi.persistence.spi.CustomObjectMapper; +import org.apache.unomi.metrics.internal.MetricsObjectMapper; @Command(scope = "metrics", name = "view", description = "This will display all the data for a single metric ") public class ViewCommand extends MetricsCommandSupport{ - @Argument(index = 0, name = "metricName", description = "The identifier for the metric", required = true, multiValued = false) + @Argument(name = "metricName", description = "The identifier for the metric", required = true) String metricName; @Override @@ -40,7 +41,7 @@ protected Object doExecute() throws Exception { // the caller values easier to read. DefaultPrettyPrinter defaultPrettyPrinter = new DefaultPrettyPrinter(); defaultPrettyPrinter = defaultPrettyPrinter.withArrayIndenter(DefaultIndenter.SYSTEM_LINEFEED_INSTANCE); - String jsonMetric = CustomObjectMapper.getObjectMapper().writer(defaultPrettyPrinter).writeValueAsString(metric); + String jsonMetric = MetricsObjectMapper.getInstance().writer(defaultPrettyPrinter).writeValueAsString(metric); System.out.println(jsonMetric); return null; } diff --git a/metrics/src/main/java/org/apache/unomi/metrics/internal/MetricsObjectMapper.java b/metrics/src/main/java/org/apache/unomi/metrics/internal/MetricsObjectMapper.java new file mode 100644 index 0000000000..35f753362c --- /dev/null +++ b/metrics/src/main/java/org/apache/unomi/metrics/internal/MetricsObjectMapper.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.metrics.internal; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +public class MetricsObjectMapper extends ObjectMapper { + + private static final MetricsObjectMapper INSTANCE = new MetricsObjectMapper(); + + private MetricsObjectMapper() { + super(); + super.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + } + + public static MetricsObjectMapper getInstance() { + return INSTANCE; + } + +} diff --git a/package/pom.xml b/package/pom.xml index 97cb7bb953..6f16886e82 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -138,23 +138,6 @@ org.apache.maven.plugins maven-dependency-plugin - - unpack-deploy-by-query-plugin - generate-resources - - unpack - - - - - org.elasticsearch.plugin - reindex-client - ${elasticsearch.version} - ${project.build.directory}/assembly/elasticsearch/modules/reindex - - - - copy package diff --git a/package/src/main/resources/etc/custom.system.properties b/package/src/main/resources/etc/custom.system.properties index 0e3cd9e5c6..3d2e57927b 100644 --- a/package/src/main/resources/etc/custom.system.properties +++ b/package/src/main/resources/etc/custom.system.properties @@ -99,18 +99,12 @@ org.apache.unomi.elasticsearch.itemTypeToRefreshPolicy=${env:UNOMI_ELASTICSEARCH org.apache.unomi.elasticsearch.fatalIllegalStateErrors=${env:UNOMI_ELASTICSEARCH_FATAL_STATE_ERRORS:-} org.apache.unomi.elasticsearch.index.prefix=${env:UNOMI_ELASTICSEARCH_INDEXPREFIX:-context} -# These monthlyIndex properties are now deprecated, please use rollover equivalent. -org.apache.unomi.elasticsearch.monthlyIndex.nbShards=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_SHARDS:-5} -org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_REPLICAS:-0} -org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_MAPPINGTOTALFIELDSLIMIT:-1000} -org.apache.unomi.elasticsearch.monthlyIndex.indexMaxDocValueFieldsSearch=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_MAXDOCVALUEFIELDSSEARCH:-1000} -org.apache.unomi.elasticsearch.monthlyIndex.itemsMonthlyIndexedOverride=${env:UNOMI_ELASTICSEARCH_MONTHLYINDEX_ITEMSMONTHLYINDEXED:-event,session} -# New rollover properties (it overrides monthlyIndex values) -org.apache.unomi.elasticsearch.rollover.nbShards=${env:UNOMI_ELASTICSEARCH_ROLLOVER_SHARDS} -org.apache.unomi.elasticsearch.rollover.nbReplicas=${env:UNOMI_ELASTICSEARCH_ROLLOVER_REPLICAS} -org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT} -org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH} -org.apache.unomi.elasticsearch.rollover.indices=${env:UNOMI_ELASTICSEARCH_ROLLOVER_INDICES} +# Rollover properties +org.apache.unomi.elasticsearch.rollover.nbShards=${env:UNOMI_ELASTICSEARCH_ROLLOVER_SHARDS:-5} +org.apache.unomi.elasticsearch.rollover.nbReplicas=${env:UNOMI_ELASTICSEARCH_ROLLOVER_REPLICAS:-0} +org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAPPINGTOTALFIELDSLIMIT:-1000} +org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXDOCVALUEFIELDSSEARCH:-1000} +org.apache.unomi.elasticsearch.rollover.indices=${env:UNOMI_ELASTICSEARCH_ROLLOVER_INDICES:-event,session} # Rollover configuration org.apache.unomi.elasticsearch.rollover.maxSize=${env:UNOMI_ELASTICSEARCH_ROLLOVER_MAXSIZE:-30gb} @@ -146,8 +140,8 @@ org.apache.unomi.elasticsearch.aggQueryMaxResponseSizeHttp=${env:UNOMI_ELASTICSE # The values used here are the default values of the API org.apache.unomi.elasticsearch.bulkProcessor.concurrentRequests=${env:UNOMI_ELASTICSEARCH_BULK_CONCURRENTREQUESTS:-1} org.apache.unomi.elasticsearch.bulkProcessor.bulkActions=${env:UNOMI_ELASTICSEARCH_BULK_ACTIONS:-1000} -org.apache.unomi.elasticsearch.bulkProcessor.bulkSize=${env:UNOMI_ELASTICSEARCH_BULK_SIZE:-5MB} -org.apache.unomi.elasticsearch.bulkProcessor.flushInterval=${env:UNOMI_ELASTICSEARCH_BULK_FLUSHINTERVAL:-5s} +org.apache.unomi.elasticsearch.bulkProcessor.bulkSize=${env:UNOMI_ELASTICSEARCH_BULK_SIZE:-5} +org.apache.unomi.elasticsearch.bulkProcessor.flushInterval=${env:UNOMI_ELASTICSEARCH_BULK_FLUSHINTERVAL:-5} org.apache.unomi.elasticsearch.bulkProcessor.backoffPolicy=${env:UNOMI_ELASTICSEARCH_BULK_BACKOFFPOLICY:-exponential} # Errors org.apache.unomi.elasticsearch.throwExceptions=${env:UNOMI_ELASTICSEARCH_THROW_EXCEPTIONS:-false} diff --git a/persistence-elasticsearch/conditions/pom.xml b/persistence-elasticsearch/conditions/pom.xml new file mode 100644 index 0000000000..cc4d5d91c0 --- /dev/null +++ b/persistence-elasticsearch/conditions/pom.xml @@ -0,0 +1,175 @@ + + + + + 4.0.0 + + + org.apache.unomi + unomi-persistence-elasticsearch + 3.0.0-SNAPSHOT + + + unomi-persistence-elasticsearch-conditions + Apache Unomi :: Persistence :: ElasticSearch :: Conditions + Conditions ElasticSearch persistence implementation for the Apache Unomi Context Server + bundle + + + + + org.apache.unomi + unomi-bom + ${project.version} + pom + import + + + + + + org.apache.unomi + unomi-api + provided + + + org.apache.unomi + unomi-common + provided + + + org.apache.unomi + unomi-persistence-spi + provided + + + org.apache.unomi + unomi-metrics + provided + + + org.apache.unomi + unomi-scripting + provided + + + org.apache.unomi + unomi-persistence-elasticsearch-core + provided + + + com.fasterxml.jackson.core + jackson-databind + + + + org.apache.commons + commons-lang3 + + + commons-collections + commons-collections + + + commons-io + commons-io + + + joda-time + joda-time + + + org.osgi + osgi.core + provided + + + co.elastic.clients + elasticsearch-java + provided + + + org.elasticsearch.client + elasticsearch-rest-client + provided + + + junit + junit + test + + + + + + + org.apache.felix + maven-bundle-plugin + true + + + + co.elastic.clients.elasticsearch._types, + co.elastic.clients.elasticsearch._types.query_dsl, + co.elastic.clients.elasticsearch._types.query_dsl.Query, + co.elastic.clients.util, + android.os;resolution:=optional, + com.conversantmedia.util.concurrent;resolution:=optional, + com.google.appengine.api;resolution:=optional, + com.google.appengine.api.utils;resolution:=optional, + com.google.apphosting.api;resolution:=optional, + jakarta.enterprise.context.spi;resolution:=optional, + jakarta.enterprise.inject.spi;resolution:=optional, + jdk.net;resolution:=optional, + org.apache.avalon.framework.logger;resolution:=optional, + org.apache.log;resolution:=optional, + org.apache.log4j, + org.brotli.dec;resolution:=optional, + org.conscrypt;resolution:=optional, + org.glassfish.hk2.osgiresourcelocator;resolution:=optional, + org.ietf.jgss;resolution:=optional, + org.joda.convert;resolution:=optional, + org.reactivestreams;resolution:=optional, + software.amazon.awssdk.auth.credentials;resolution:=optional, + software.amazon.awssdk.core.async;resolution:=optional, + software.amazon.awssdk.http;resolution:=optional, + software.amazon.awssdk.http.async;resolution:=optional, + software.amazon.awssdk.http.auth.aws.signer;resolution:=optional, + software.amazon.awssdk.http.auth.spi.signer;resolution:=optional, + software.amazon.awssdk.identity.spi;resolution:=optional, + software.amazon.awssdk.regions;resolution:=optional, + software.amazon.awssdk.utils;resolution:=optional, + software.amazon.awssdk.utils.builder;resolution:=optional, + sun.misc;resolution:=optional, + sun.nio.ch;resolution:=optional, + io.opentelemetry.sdk.autoconfigure;resolution:=optional, + jakarta.json.bind;resolution:=optional, + jakarta.json.bind.spi;resolution:=optional, + * + + *;scope=compile|runtime + true + + + + + + + + diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java b/persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/GeoLocationByPointSessionConditionESQueryBuilder.java similarity index 64% rename from plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java rename to persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/GeoLocationByPointSessionConditionESQueryBuilder.java index 21d27b1270..15df3d2d97 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionESQueryBuilder.java +++ b/persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/GeoLocationByPointSessionConditionESQueryBuilder.java @@ -15,19 +15,18 @@ * limitations under the License. */ -package org.apache.unomi.plugins.baseplugin.conditions; +package org.apache.unomi.persistence.elasticsearch.querybuilders.advanced; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import java.util.Map; public class GeoLocationByPointSessionConditionESQueryBuilder implements ConditionESQueryBuilder { @Override - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { String type = (String) condition.getParameter("type"); String name = condition.getParameter("name") == null ? "location" : (String) condition.getParameter("name"); @@ -37,9 +36,7 @@ public QueryBuilder buildQuery(Condition condition, Map context, String distance = condition.getParameter("distance").toString(); if(circleLatitude != null && circleLongitude != null && distance != null) { - return QueryBuilders.geoDistanceQuery(name) - .point(circleLatitude, circleLongitude) - .distance(distance); + return Query.of(q -> q.geoDistance(g -> g.field(name).location(l -> l.latlon(latlong -> latlong.lat(circleLatitude).lon(circleLongitude))).distance(distance))); } } else if("rectangle".equals(type)) { Double rectLatitudeNE = (Double) condition.getParameter("rectLatitudeNE"); @@ -48,8 +45,18 @@ public QueryBuilder buildQuery(Condition condition, Map context, Double rectLongitudeSW = (Double) condition.getParameter("rectLongitudeSW"); if(rectLatitudeNE != null && rectLongitudeNE != null && rectLatitudeSW != null && rectLongitudeSW != null) { - return QueryBuilders.geoBoundingBoxQuery(name) - .setCorners(rectLatitudeNE, rectLongitudeNE,rectLatitudeSW, rectLongitudeSW); + return Query.of(q -> q.geoBoundingBox(g -> g + .field(name) + .boundingBox(b -> b + .coords(c -> c + .top(rectLatitudeNE) + .left(rectLongitudeNE) + .bottom(rectLatitudeSW) + .right(rectLongitudeSW) + ) + ) + ) + ); } } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/IdsConditionESQueryBuilder.java b/persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/IdsConditionESQueryBuilder.java similarity index 64% rename from plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/IdsConditionESQueryBuilder.java rename to persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/IdsConditionESQueryBuilder.java index d535c1b650..50dc9f55fa 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/IdsConditionESQueryBuilder.java +++ b/persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/IdsConditionESQueryBuilder.java @@ -14,15 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.unomi.plugins.baseplugin.conditions; +package org.apache.unomi.persistence.elasticsearch.querybuilders.advanced; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.IdsQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import java.util.Collection; import java.util.Map; @@ -35,8 +32,9 @@ public void setMaximumIdsQueryCount(int maximumIdsQueryCount) { this.maximumIdsQueryCount = maximumIdsQueryCount; } + @Override - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { Collection ids = (Collection) condition.getParameter("ids"); Boolean match = (Boolean) condition.getParameter("match"); @@ -45,13 +43,11 @@ public QueryBuilder buildQuery(Condition condition, Map context, throw new UnsupportedOperationException("Too many profiles"); } - IdsQueryBuilder idsQueryBuilder = QueryBuilders.idsQuery().addIds(ids.toArray(new String[0])); + Query idsQuery = Query.of(q -> q.ids(i -> i.values(ids.stream().toList()))); if (match) { - return idsQueryBuilder; + return idsQuery; } else { - BoolQueryBuilder boolQuery = QueryBuilders.boolQuery(); - boolQuery.mustNot(idsQueryBuilder); - return boolQuery; + return Query.of(q -> q.bool(b -> b.mustNot(idsQuery))); } } } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java b/persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/PastEventConditionESQueryBuilder.java similarity index 89% rename from plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java rename to persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/PastEventConditionESQueryBuilder.java index 6c40f9f874..2fc8bfa078 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionESQueryBuilder.java +++ b/persistence-elasticsearch/conditions/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/advanced/PastEventConditionESQueryBuilder.java @@ -15,26 +15,30 @@ * limitations under the License. */ -package org.apache.unomi.plugins.baseplugin.conditions; +package org.apache.unomi.persistence.elasticsearch.querybuilders.advanced; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.Event; import org.apache.unomi.api.Profile; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.services.DefinitionsService; import org.apache.unomi.api.services.SegmentService; import org.apache.unomi.api.utils.ConditionBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionContextHelper; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.TermsAggregate; +import org.apache.unomi.persistence.spi.conditions.ConditionContextHelper; +import org.apache.unomi.persistence.spi.conditions.PastEventConditionPersistenceQueryBuilder; import org.apache.unomi.scripting.ScriptExecutor; -import org.elasticsearch.index.query.*; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; import java.util.*; import java.util.stream.Collectors; -public class PastEventConditionESQueryBuilder implements ConditionESQueryBuilder { +public class PastEventConditionESQueryBuilder implements ConditionESQueryBuilder, PastEventConditionPersistenceQueryBuilder { private DefinitionsService definitionsService; private PersistenceService persistenceService; @@ -45,6 +49,8 @@ public class PastEventConditionESQueryBuilder implements ConditionESQueryBuilder private int aggregateQueryBucketSize = 5000; private boolean pastEventsDisablePartitions = false; + private final DateTimeFormatter dateTimeFormatter = ISODateTimeFormat.dateTime(); + public void setDefinitionsService(DefinitionsService definitionsService) { this.definitionsService = definitionsService; } @@ -74,7 +80,7 @@ public void setSegmentService(SegmentService segmentService) { } @Override - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { boolean eventsOccurred = getStrategyFromOperator((String) condition.getParameter("operator")); int minimumEventCount = !eventsOccurred || condition.getParameter("minimumEventCount") == null ? 1 : (Integer) condition.getParameter("minimumEventCount"); int maximumEventCount = !eventsOccurred || condition.getParameter("maximumEventCount") == null ? Integer.MAX_VALUE : (Integer) condition.getParameter("maximumEventCount"); @@ -117,7 +123,7 @@ public long count(Condition condition, Map context, ConditionESQ } } - protected static boolean getStrategyFromOperator(String operator) { + public boolean getStrategyFromOperator(String operator) { if (operator != null && !operator.equals("eventsOccurred") && !operator.equals("eventsNotOccurred")) { throw new UnsupportedOperationException("Unsupported operator: " + operator + ", please use either 'eventsOccurred' or 'eventsNotOccurred'"); } @@ -198,7 +204,7 @@ private Set getProfileIdsMatchingEventCount(Condition eventCondition, in } } - protected static Condition getEventCondition(Condition condition, Map context, String profileId, + public Condition getEventCondition(Condition condition, Map context, String profileId, DefinitionsService definitionsService, ScriptExecutor scriptExecutor) { Condition eventCondition; try { @@ -227,8 +233,24 @@ protected static Condition getEventCondition(Condition condition, Map + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder + org.apache.unomi.persistence.spi.conditions.PastEventConditionPersistenceQueryBuilder + + + + + + + + + + + + + + + + diff --git a/persistence-elasticsearch/core/pom.xml b/persistence-elasticsearch/core/pom.xml index e9c990b3d7..7a8beb4d1f 100644 --- a/persistence-elasticsearch/core/pom.xml +++ b/persistence-elasticsearch/core/pom.xml @@ -23,11 +23,13 @@ unomi-persistence-elasticsearch 3.0.0-SNAPSHOT + unomi-persistence-elasticsearch-core Apache Unomi :: Persistence :: ElasticSearch :: Core Core ElasticSearch persistence implementation for the Apache Unomi Context Server bundle + @@ -79,9 +81,12 @@ - com.hazelcast - hazelcast-all - provided + co.elastic.clients + elasticsearch-java + + + org.elasticsearch.client + elasticsearch-rest-client org.slf4j @@ -94,6 +99,11 @@ jackson-databind provided + + com.fasterxml.jackson.core + jackson-core + provided + org.apache.commons commons-lang3 @@ -106,10 +116,10 @@ - org.elasticsearch.client - elasticsearch-rest-high-level-client + joda-time + joda-time + provided - org.apache.logging.log4j @@ -152,16 +162,6 @@ junit test - - org.elasticsearch.test - framework - test - - - org.elasticsearch.plugin - transport-netty4-client - test - org.apache.lucene lucene-test-framework @@ -197,7 +197,6 @@ com.fasterxml.jackson.*;resolution:=optional, com.google.appengine.api;resolution:=optional, com.google.apphosting.api;resolution:=optional, - com.google.common.geometry;resolution:=optional, com.google.errorprone.annotations.concurrent;resolution:=optional, com.lmax.disruptor;resolution:=optional, com.lmax.disruptor.dsl;resolution:=optional, @@ -246,6 +245,7 @@ org.apache.unomi.metrics, org.apache.unomi.persistence.spi.aggregate, org.apache.unomi.persistence.spi, + org.apache.unomi.persistence.spi.conditions, org.codehaus.stax2;resolution:=optional, org.elasticsearch.*;resolution:=optional, org.ietf.jgss;resolution:=optional, @@ -261,13 +261,34 @@ org.xml.sax.ext;resolution:=optional, org.xml.sax.helpers;resolution:=optional, org.zeromq;resolution:=optional, + org.conscrypt;resolution:=optional, + org.glassfish.hk2.osgiresourcelocator;resolution:=optional, + org.brotli.dec;resolution:=optional, + io.opentelemetry.opentelemetry-api;resolution:=optional, + io.opentelemetry.sdk.autoconfigure;resolution:=optional, + io.opentelemetry.opentelemetry-context;resolution:=optional, + io.opentelemetry.opentelemetry-sdk;resolution:=optional, + io.opentelemetry.opentelemetry-sdk-extension-autoconfigure;resolution:=optional, + org.apache.httpcomponents.client5.httpclient5;resolution:=optional, + org.apache.httpcomponents.core5.httpcore5;resolution:=optional, + org.apache.httpcomponents.core5.httpcore5-h2;resolution:=optional, + jakarta.json.bind;resolution:=optional, + jakarta.json.bind.spi;resolution:=optional, + jakarta.json.spi;resolution:=optional, + jakarta.json.stream;resolution:=optional, + org.eclipse.parsson.parsson;resolution:=optional, + org.eclipse.yasson;resolution:=optional, + org.apache.http.impl.nio.client;resolution:=optional, * - org.elasticsearch.*;version="${elasticsearch.version}", - org.elasticsearch.index.query.*;version="${elasticsearch.version}", org.apache.lucene.search.join.*;version="${lucene.version}", - org.apache.unomi.persistence.elasticsearch.conditions;version="${project.version}" + org.apache.unomi.persistence.elasticsearch;version="${project.version}", + co.elastic.clients.elasticsearch;version="${elasticsearch.version}", + co.elastic.clients.elasticsearch._types.query_dsl.Query;version="${elasticsearch.version}", + co.elastic.clients.elasticsearch._types.query_dsl;version="${elasticsearch.version}", + co.elastic.clients.util;version="${elasticsearch.version}", + co.elastic.clients.elasticsearch._types;version="${elasticsearch.version}" *;scope=compile|runtime true @@ -302,4 +323,4 @@ - + \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ChildFirstClassLoader.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ChildFirstClassLoader.java deleted file mode 100644 index 8981df18a4..0000000000 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ChildFirstClassLoader.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.unomi.persistence.elasticsearch; -import java.net.URL; -import java.net.URLClassLoader; - -/** - * This class loader will always try to load classes first from the child URL class loader and will only resort to the - * parent class loader if the class coudln't be found. - */ -public class ChildFirstClassLoader extends ClassLoader { - - private ChildFirstURLClassLoader childFirstURLClassLoader; - - private static class ChildFirstURLClassLoader extends URLClassLoader { - private ClassLoader parentClassLoader; - - public ChildFirstURLClassLoader(URL[] urls, ClassLoader parentClassLoader) { - super(urls, null); - - this.parentClassLoader = parentClassLoader; - } - - @Override - public Class findClass(String name) throws ClassNotFoundException { - try { - return super.findClass(name); - } catch (ClassNotFoundException e) { - return parentClassLoader.loadClass(name); - } - } - } - - public ChildFirstClassLoader(ClassLoader parent, URL[] urls) { - super(parent); - childFirstURLClassLoader = new ChildFirstURLClassLoader(urls, parent); - } - - @Override - protected synchronized Class loadClass(String name, boolean resolve) - throws ClassNotFoundException { - try { - return childFirstURLClassLoader.loadClass(name); - } catch (ClassNotFoundException e) { - return super.loadClass(name, resolve); - } - } - -} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ConditionESQueryBuilder.java similarity index 82% rename from persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java rename to persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ConditionESQueryBuilder.java index 7a100d05a6..7fc1f3a34c 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilder.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ConditionESQueryBuilder.java @@ -15,16 +15,16 @@ * limitations under the License. */ -package org.apache.unomi.persistence.elasticsearch.conditions; +package org.apache.unomi.persistence.elasticsearch; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; -import org.elasticsearch.index.query.QueryBuilder; import java.util.Map; public interface ConditionESQueryBuilder { - QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher); + Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher); default long count(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { throw new UnsupportedOperationException(); diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ConditionESQueryBuilderDispatcher.java similarity index 85% rename from persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java rename to persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ConditionESQueryBuilderDispatcher.java index 010811549d..39d7963599 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionESQueryBuilderDispatcher.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ConditionESQueryBuilderDispatcher.java @@ -15,12 +15,12 @@ * limitations under the License. */ -package org.apache.unomi.persistence.elasticsearch.conditions; +package org.apache.unomi.persistence.elasticsearch; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; +import org.apache.unomi.persistence.spi.conditions.ConditionContextHelper; import org.apache.unomi.scripting.ScriptExecutor; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,16 +54,18 @@ public String getQuery(Condition condition) { return "{\"query\": " + getQueryBuilder(condition).toString() + "}"; } - public QueryBuilder getQueryBuilder(Condition condition) { - return QueryBuilders.boolQuery().must(QueryBuilders.matchAllQuery()).filter(buildFilter(condition)); + public Query getQueryBuilder(Condition condition) { + Query.Builder qb = new Query.Builder(); + return qb.bool(b -> b.must(Query.of(q -> q.matchAll(m -> m))).filter(buildFilter(condition))).build(); + } - public QueryBuilder buildFilter(Condition condition) { - return buildFilter(condition, new HashMap()); + public Query buildFilter(Condition condition) { + return buildFilter(condition, new HashMap<>()); } - public QueryBuilder buildFilter(Condition condition, Map context) { - if(condition == null || condition.getConditionType() == null) { + public Query buildFilter(Condition condition, Map context) { + if (condition == null || condition.getConditionType() == null) { throw new IllegalArgumentException("Condition is null or doesn't have type, impossible to build filter"); } @@ -89,7 +91,7 @@ public QueryBuilder buildFilter(Condition condition, Map context LOGGER.debug("No matching query builder for condition {} and context {}", condition, context); } - return QueryBuilders.matchAllQuery(); + return Query.of(q -> q.matchAll(m -> m)); } public long count(Condition condition) { @@ -97,7 +99,7 @@ public long count(Condition condition) { } public long count(Condition condition, Map context) { - if(condition == null || condition.getConditionType() == null) { + if (condition == null || condition.getConditionType() == null) { throw new IllegalArgumentException("Condition is null or doesn't have type, impossible to build filter"); } diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESCustomObjectMapper.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESCustomObjectMapper.java index 0048398f91..7403658c00 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESCustomObjectMapper.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESCustomObjectMapper.java @@ -16,14 +16,17 @@ */ package org.apache.unomi.persistence.elasticsearch; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.unomi.api.Event; import org.apache.unomi.api.Item; import org.apache.unomi.persistence.spi.CustomObjectMapper; +import java.util.Map; + /** - * This CustomObjectMapper is used to avoid the version parameter to be registered in ES - * @author dgaillard + * This CustomObjectMapper is used to avoid the version parameter to be registered in Elasticsearch */ public class ESCustomObjectMapper extends CustomObjectMapper { @@ -31,8 +34,11 @@ public class ESCustomObjectMapper extends CustomObjectMapper { public ESCustomObjectMapper() { super(); + super.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); this.addMixIn(Item.class, ESItemMixIn.class); this.addMixIn(Event.class, ESEventMixIn.class); + this.configOverride(Map.class) + .setInclude(JsonInclude.Value.construct(JsonInclude.Include.ALWAYS, JsonInclude.Include.ALWAYS)); } public static ObjectMapper getObjectMapper() { diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESItemMixIn.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESItemMixIn.java index c20d8ef9dd..839f181d53 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESItemMixIn.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ESItemMixIn.java @@ -20,7 +20,6 @@ /** * This mixin is used in ESCustomObjectMapper to avoid the version parameter to be registered in ES - * @author dgaillard */ public abstract class ESItemMixIn { diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java index 372fb6950e..624a721ed5 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java @@ -14,22 +14,54 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.unomi.persistence.elasticsearch; +import co.elastic.clients.elasticsearch._helpers.bulk.BulkIngester; +import co.elastic.clients.elasticsearch._helpers.bulk.BulkListener; +import co.elastic.clients.elasticsearch._types.*; +import co.elastic.clients.elasticsearch._types.aggregations.*; +import co.elastic.clients.elasticsearch._types.analysis.CustomAnalyzer; +import co.elastic.clients.elasticsearch._types.mapping.Property; +import co.elastic.clients.elasticsearch._types.mapping.TypeMapping; +import co.elastic.clients.elasticsearch._types.query_dsl.*; +import co.elastic.clients.elasticsearch.core.*; +import co.elastic.clients.elasticsearch.core.CountRequest; +import co.elastic.clients.elasticsearch.core.DeleteRequest; +import co.elastic.clients.elasticsearch.core.SearchRequest; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.bulk.BulkOperation; +import co.elastic.clients.elasticsearch.core.bulk.UpdateAction; +import co.elastic.clients.elasticsearch.core.bulk.UpdateOperation; +import co.elastic.clients.elasticsearch.core.search.Hit; +import co.elastic.clients.elasticsearch.core.search.TotalHits; +import co.elastic.clients.elasticsearch.core.search.TotalHitsRelation; +import co.elastic.clients.elasticsearch.ilm.*; +import co.elastic.clients.elasticsearch.indices.*; +import co.elastic.clients.elasticsearch.indices.Alias; +import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; +import co.elastic.clients.elasticsearch.indices.DeleteIndexTemplateRequest; +import co.elastic.clients.elasticsearch.indices.ExistsRequest; +import co.elastic.clients.elasticsearch.indices.RefreshRequest; +import co.elastic.clients.elasticsearch.indices.get_alias.IndexAliases; +import co.elastic.clients.elasticsearch.indices.get_mapping.IndexMappingRecord; +import co.elastic.clients.elasticsearch.indices.put_index_template.IndexTemplateMapping; +import co.elastic.clients.elasticsearch.tasks.GetTasksRequest; +import co.elastic.clients.elasticsearch.tasks.GetTasksResponse; +import co.elastic.clients.json.JsonData; +import co.elastic.clients.json.JsonpMapper; +import co.elastic.clients.transport.BackoffPolicy; +import co.elastic.clients.transport.endpoints.BooleanResponse; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.transport.rest_client.RestClientOptions; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.json.stream.JsonGenerator; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.log4j.Level; -import org.apache.lucene.search.TotalHits; import org.apache.unomi.api.*; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.query.DateRange; @@ -37,93 +69,23 @@ import org.apache.unomi.api.query.NumericRange; import org.apache.unomi.metrics.MetricAdapter; import org.apache.unomi.metrics.MetricsService; -import org.apache.unomi.persistence.elasticsearch.conditions.*; import org.apache.unomi.persistence.spi.PersistenceService; import org.apache.unomi.persistence.spi.aggregate.*; -import org.apache.unomi.persistence.spi.config.ConfigurationUpdateHelper; -import org.elasticsearch.ElasticsearchStatusException; -import org.elasticsearch.Version; -import org.elasticsearch.action.DocWriteRequest; -import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; -import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest; -import org.elasticsearch.action.admin.indices.alias.Alias; -import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest; -import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; -import org.elasticsearch.action.admin.indices.refresh.RefreshRequest; -import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateRequest; -import org.elasticsearch.action.bulk.*; -import org.elasticsearch.action.delete.DeleteRequest; -import org.elasticsearch.action.get.GetRequest; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.action.search.ClearScrollRequest; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchScrollRequest; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.action.support.master.AcknowledgedResponse; -import org.elasticsearch.action.update.UpdateRequest; +import org.apache.unomi.persistence.spi.aggregate.DateRangeAggregate; +import org.apache.unomi.persistence.spi.aggregate.IpRangeAggregate; +import org.apache.unomi.persistence.spi.conditions.ConditionContextHelper; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; import org.elasticsearch.client.*; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.client.core.CountResponse; -import org.elasticsearch.client.core.MainResponse; -import org.elasticsearch.client.indexlifecycle.*; -import org.elasticsearch.client.indices.*; -import org.elasticsearch.client.tasks.GetTaskRequest; -import org.elasticsearch.client.tasks.GetTaskResponse; -import org.elasticsearch.client.tasks.TaskSubmissionResponse; -import org.elasticsearch.cluster.metadata.AliasMetaData; -import org.elasticsearch.cluster.metadata.MappingMetaData; -import org.elasticsearch.common.bytes.BytesReference; -import org.elasticsearch.common.unit.ByteSizeUnit; -import org.elasticsearch.common.unit.ByteSizeValue; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.*; -import org.elasticsearch.index.IndexNotFoundException; -import org.elasticsearch.index.query.*; -import org.elasticsearch.index.reindex.*; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.script.Script; -import org.elasticsearch.script.ScriptException; -import org.elasticsearch.script.ScriptType; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.*; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; -import org.elasticsearch.search.aggregations.bucket.SingleBucketAggregation; -import org.elasticsearch.search.aggregations.bucket.filter.Filter; -import org.elasticsearch.search.aggregations.bucket.global.Global; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; -import org.elasticsearch.search.aggregations.bucket.missing.MissingAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.range.IpRangeAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude; -import org.elasticsearch.search.aggregations.bucket.terms.Terms; -import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.sort.GeoDistanceSortBuilder; -import org.elasticsearch.search.sort.SortBuilders; -import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.tasks.TaskId; + import org.osgi.framework.*; -import org.osgi.service.cm.ManagedService; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.*; import java.net.URL; import java.nio.charset.StandardCharsets; import java.security.KeyManagementException; @@ -132,15 +94,10 @@ import java.security.cert.X509Certificate; import java.util.*; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; - -@SuppressWarnings("rawtypes") -public class ElasticSearchPersistenceServiceImpl implements PersistenceService, SynchronousBundleListener, ManagedService { +public class ElasticSearchPersistenceServiceImpl implements PersistenceService, SynchronousBundleListener { - public static final String BULK_PROCESSOR_BULK_SIZE = "bulkProcessor.bulkSize"; - public static final String BULK_PROCESSOR_FLUSH_INTERVAL = "bulkProcessor.flushInterval"; - public static final String BULK_PROCESSOR_BACKOFF_POLICY = "bulkProcessor.backoffPolicy"; public static final String SEQ_NO = "seq_no"; public static final String PRIMARY_TERM = "primary_term"; @@ -148,37 +105,31 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private static final String ROLLOVER_LIFECYCLE_NAME = "unomi-rollover-policy"; private boolean throwExceptions = false; - private CustomRestHighLevelClient client; - private BulkProcessor bulkProcessor; + private ElasticsearchClient esClient; + private BulkIngester bulkIngester; private String elasticSearchAddresses; - private List elasticSearchAddressList = new ArrayList<>(); - private String clusterName; + private final List elasticSearchAddressList = new ArrayList<>(); private String indexPrefix; - private String monthlyIndexNumberOfShards; - private String monthlyIndexNumberOfReplicas; - private String monthlyIndexMappingTotalFieldsLimit; - private String monthlyIndexMaxDocValueFieldsSearch; private String numberOfShards; private String numberOfReplicas; private String indexMappingTotalFieldsLimit; private String indexMaxDocValueFieldsSearch; private String[] fatalIllegalStateErrors; private BundleContext bundleContext; - private Map mappings = new HashMap(); + private final Map mappings = new HashMap(); private ConditionEvaluatorDispatcher conditionEvaluatorDispatcher; private ConditionESQueryBuilderDispatcher conditionESQueryBuilderDispatcher; - private List itemsMonthlyIndexed; private Map routingByType; private Integer defaultQueryLimit = 10; - private Integer removeByQueryTimeoutInMinutes = 10; + private final Integer removeByQueryTimeoutInMinutes = 10; private Integer taskWaitingTimeout = 3600000; private Integer taskWaitingPollingInterval = 1000; private String bulkProcessorConcurrentRequests = "1"; private String bulkProcessorBulkActions = "1000"; - private String bulkProcessorBulkSize = "5MB"; - private String bulkProcessorFlushInterval = "5s"; + private Long bulkProcessorBulkSize = 5L; + private Long bulkProcessorFlushIntervalInSeconds = 5L; private String bulkProcessorBackoffPolicy = "exponential"; // Rollover configuration @@ -192,8 +143,8 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private String rolloverIndexMappingTotalFieldsLimit; private String rolloverIndexMaxDocValueFieldsSearch; - private String minimalElasticSearchVersion = "7.0.0"; - private String maximalElasticSearchVersion = "8.0.0"; + private String minimalElasticSearchVersion = "9.0.3"; + private String maximalElasticSearchVersion = "10.0.0"; // authentication props private String username; @@ -211,14 +162,15 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService, private boolean aggQueryThrowOnMissingDocs = false; private Integer aggQueryMaxResponseSizeHttp = null; private Integer clientSocketTimeout = null; - private Map itemTypeToRefreshPolicy = new HashMap<>(); + private Map itemTypeToRefreshPolicy = new HashMap<>(); - private Map>> knownMappings = new HashMap<>(); + private final Map>> knownMappings = new HashMap<>(); private static final Map itemTypeIndexNameMap = new HashMap<>(); - private static final Collection systemItems = Arrays.asList("actionType", "campaign", "campaignevent", "goal", - "userList", "propertyType", "scope", "conditionType", "rule", "scoring", "segment", "groovyAction", "topic", - "patch", "jsonSchema", "importConfig", "exportConfig", "rulestats"); + private static final Collection systemItems = Arrays.asList("actionType", "campaign", "campaignevent", "goal", "userList", + "propertyType", "scope", "conditionType", "rule", "scoring", "segment", "groovyAction", "topic", "patch", "jsonSchema", + "importConfig", "exportConfig", "rulestats"); + static { for (String systemItem : systemItems) { itemTypeIndexNameMap.put(systemItem, "systemItems"); @@ -232,10 +184,6 @@ public void setBundleContext(BundleContext bundleContext) { this.bundleContext = bundleContext; } - public void setClusterName(String clusterName) { - this.clusterName = clusterName; - } - public void setElasticSearchAddresses(String elasticSearchAddresses) { this.elasticSearchAddresses = elasticSearchAddresses; String[] elasticSearchAddressesArray = elasticSearchAddresses.split(","); @@ -248,14 +196,14 @@ public void setElasticSearchAddresses(String elasticSearchAddresses) { public void setItemTypeToRefreshPolicy(String itemTypeToRefreshPolicy) throws IOException { if (!itemTypeToRefreshPolicy.isEmpty()) { this.itemTypeToRefreshPolicy = new ObjectMapper().readValue(itemTypeToRefreshPolicy, - new TypeReference>() { + new TypeReference>() { }); } } public void setFatalIllegalStateErrors(String fatalIllegalStateErrors) { - this.fatalIllegalStateErrors = Arrays.stream(fatalIllegalStateErrors.split(",")) - .map(i -> i.trim()).filter(i -> !i.isEmpty()).toArray(String[]::new); + this.fatalIllegalStateErrors = Arrays.stream(fatalIllegalStateErrors.split(",")).map(i -> i.trim()).filter(i -> !i.isEmpty()) + .toArray(String[]::new); } public void setAggQueryMaxResponseSizeHttp(String aggQueryMaxResponseSizeHttp) { @@ -268,31 +216,6 @@ public void setIndexPrefix(String indexPrefix) { this.indexPrefix = indexPrefix; } - @Deprecated - public void setMonthlyIndexNumberOfShards(String monthlyIndexNumberOfShards) { - this.monthlyIndexNumberOfShards = monthlyIndexNumberOfShards; - } - - @Deprecated - public void setMonthlyIndexNumberOfReplicas(String monthlyIndexNumberOfReplicas) { - this.monthlyIndexNumberOfReplicas = monthlyIndexNumberOfReplicas; - } - - @Deprecated - public void setMonthlyIndexMappingTotalFieldsLimit(String monthlyIndexMappingTotalFieldsLimit) { - this.monthlyIndexMappingTotalFieldsLimit = monthlyIndexMappingTotalFieldsLimit; - } - - @Deprecated - public void setMonthlyIndexMaxDocValueFieldsSearch(String monthlyIndexMaxDocValueFieldsSearch) { - this.monthlyIndexMaxDocValueFieldsSearch = monthlyIndexMaxDocValueFieldsSearch; - } - - @Deprecated - public void setItemsMonthlyIndexedOverride(String itemsMonthlyIndexedOverride) { - this.itemsMonthlyIndexed = StringUtils.isNotEmpty(itemsMonthlyIndexedOverride) ? Arrays.asList(itemsMonthlyIndexedOverride.split(",").clone()) : Collections.emptyList(); - } - public void setNumberOfShards(String numberOfShards) { this.numberOfShards = numberOfShards; } @@ -333,12 +256,12 @@ public void setBulkProcessorBulkActions(String bulkProcessorBulkActions) { this.bulkProcessorBulkActions = bulkProcessorBulkActions; } - public void setBulkProcessorBulkSize(String bulkProcessorBulkSize) { + public void setBulkProcessorBulkSize(Long bulkProcessorBulkSize) { this.bulkProcessorBulkSize = bulkProcessorBulkSize; } - public void setBulkProcessorFlushInterval(String bulkProcessorFlushInterval) { - this.bulkProcessorFlushInterval = bulkProcessorFlushInterval; + public void setBulkProcessorFlushIntervalInSeconds(Long bulkProcessorFlushIntervalInSeconds) { + this.bulkProcessorFlushIntervalInSeconds = bulkProcessorFlushIntervalInSeconds; } public void setBulkProcessorBackoffPolicy(String bulkProcessorBackoffPolicy) { @@ -423,7 +346,6 @@ public void setSslTrustAllCertificates(boolean sslTrustAllCertificates) { this.sslTrustAllCertificates = sslTrustAllCertificates; } - public void setAggQueryThrowOnMissingDocs(boolean aggQueryThrowOnMissingDocs) { this.aggQueryThrowOnMissingDocs = aggQueryThrowOnMissingDocs; } @@ -452,31 +374,65 @@ public void setTaskWaitingPollingInterval(String taskWaitingPollingInterval) { } } + /** + * Check if the current cluster version is in the expected range + * + * @return true if the version of the current elasticsearch is not in the expected range + */ + private boolean versionIsNotCompatible() throws IOException { + InfoResponse info = esClient.info(); + String currentVersion = info.version().number(); + + return compareVersions(currentVersion, minimalElasticSearchVersion) < 0 + || compareVersions(currentVersion, maximalElasticSearchVersion) >= 0; + } + + /** + * Compare to semantic versions + * + * @param version1 First version + * @param version2 Second vrsion + * @return positive if version1 > version2, 0 if equals, negative if version1 < version2 + */ + private static int compareVersions(String version1, String version2) { + String[] parts1 = version1.split("\\."); + String[] parts2 = version2.split("\\."); + + int length = Math.max(parts1.length, parts2.length); + + for (int i = 0; i < length; i++) { + int part1 = i < parts1.length ? Integer.parseInt(parts1[i]) : 0; + int part2 = i < parts2.length ? Integer.parseInt(parts2[i]) : 0; + + if (part1 != part2) { + return part1 - part2; + } + } + + return 0; + } + public void start() throws Exception { // Work around to avoid ES Logs regarding the deprecated [ignore_throttled] parameter try { Level lvl = Level.toLevel(logLevelRestClient, Level.ERROR); + //TODO ensure this is necessary org.apache.log4j.Logger.getLogger("org.elasticsearch.client.RestClient").setLevel(lvl); } catch (Exception e) { // Never fail because of the set of the logger } // on startup - new InClassLoaderExecute(null, null, this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + new InClassLoaderExecute<>(null, null, this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { public Object execute(Object... args) throws Exception { buildClient(); - MainResponse response = client.info(RequestOptions.DEFAULT); - MainResponse.Version version = response.getVersion(); - Version clusterVersion = Version.fromString(version.getNumber()); - Version minimalVersion = Version.fromString(minimalElasticSearchVersion); - Version maximalVersion = Version.fromString(maximalElasticSearchVersion); - if (clusterVersion.before(minimalVersion) || - clusterVersion.equals(maximalVersion) || - clusterVersion.after(maximalVersion)) { - throw new Exception("ElasticSearch version is not within [" + minimalVersion + "," + maximalVersion + "), aborting startup !"); + if (versionIsNotCompatible()) { + throw new Exception( + "ElasticSearch version is not within [" + minimalElasticSearchVersion + "," + maximalElasticSearchVersion + + "), aborting startup !"); } registerRolloverLifecyclePolicy(); @@ -492,20 +448,16 @@ public Object execute(Object... args) throws Exception { } } - if (client != null && bulkProcessor == null) { - bulkProcessor = getBulkProcessor(); - } - // Wait for green LOGGER.info("Waiting for GREEN cluster status..."); - client.cluster().health(new ClusterHealthRequest().waitForGreenStatus(), RequestOptions.DEFAULT); + esClient.cluster().health(builder -> builder.waitForStatus(HealthStatus.Green)); LOGGER.info("Cluster status is GREEN"); // We keep in memory the latest available session index to be able to load session using direct GET access on ES if (isItemTypeRollingOver(Session.ITEM_TYPE)) { LOGGER.info("Sessions are using rollover indices, loading latest session index available ..."); - GetAliasesResponse sessionAliasResponse = client.indices().getAlias(new GetAliasesRequest(getIndex(Session.ITEM_TYPE)), RequestOptions.DEFAULT); - Map> aliases = sessionAliasResponse.getAliases(); + GetAliasResponse getAliasResponse = esClient.indices().getAlias(builder -> builder.name(getIndex(Session.ITEM_TYPE))); + Map aliases = getAliasResponse.aliases(); if (!aliases.isEmpty()) { sessionLatestIndex = new TreeSet<>(aliases.keySet()).last(); LOGGER.info("Latest available session index found is: {}", sessionLatestIndex); @@ -523,160 +475,134 @@ public Object execute(Object... args) throws Exception { LOGGER.info("{} service started successfully.", this.getClass().getName()); } - private void buildClient() throws NoSuchFieldException, IllegalAccessException { - List nodeList = new ArrayList<>(); + private List getHosts() { + List hosts = new ArrayList<>(); for (String elasticSearchAddress : elasticSearchAddressList) { String[] elasticSearchAddressParts = elasticSearchAddress.split(":"); String elasticSearchHostName = elasticSearchAddressParts[0]; int elasticSearchPort = Integer.parseInt(elasticSearchAddressParts[1]); - - // configure authentication - nodeList.add(new Node(new HttpHost(elasticSearchHostName, elasticSearchPort, sslEnable ? "https" : "http"))); + hosts.add(new HttpHost(elasticSearchHostName, elasticSearchPort, sslEnable ? "https" : "http")); } + return hosts; + } - RestClientBuilder clientBuilder = RestClient.builder(nodeList.toArray(new Node[nodeList.size()])); - - if (clientSocketTimeout != null) { - clientBuilder.setRequestConfigCallback(requestConfigBuilder -> { - requestConfigBuilder.setSocketTimeout(clientSocketTimeout); - return requestConfigBuilder; - }); + private void buildClient() throws NoSuchFieldException, IllegalAccessException { + final SSLContext sslContext; + try { + sslContext = SSLContext.getInstance("SSL"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); } - clientBuilder.setHttpClientConfigCallback(httpClientBuilder -> { - if (sslTrustAllCertificates) { - try { - final SSLContext sslContext = SSLContext.getInstance("SSL"); - sslContext.init(null, new TrustManager[]{new X509TrustManager() { - public X509Certificate[] getAcceptedIssuers() { - return null; - } - - public void checkClientTrusted(X509Certificate[] certs, - String authType) { - } - - public void checkServerTrusted(X509Certificate[] certs, - String authType) { - } - }}, new SecureRandom()); - - httpClientBuilder.setSSLContext(sslContext).setSSLHostnameVerifier(new NoopHostnameVerifier()); - } catch (NoSuchAlgorithmException | KeyManagementException e) { - LOGGER.error("Error creating SSL Context for trust all certificates", e); - } - } + if (sslTrustAllCertificates) { + try { + sslContext.init(null, new TrustManager[] { new X509TrustManager() { + public X509Certificate[] getAcceptedIssuers() { + return null; + } - if (StringUtils.isNotBlank(username)) { - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + public void checkClientTrusted(X509Certificate[] certs, String authType) { + } - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + public void checkServerTrusted(X509Certificate[] certs, String authType) { + } + } }, new SecureRandom()); + } catch (KeyManagementException e) { + LOGGER.error("Error creating SSL Context for trust all certificates", e); } + } - return httpClientBuilder; - }); + esClient = ElasticsearchClientFactory.builder().hosts(getHosts()).socketTimeout(clientSocketTimeout).sslContext(sslContext) + .usernameAndPassword(username, password).build(); - LOGGER.info("Connecting to ElasticSearch persistence backend using cluster name {} and index prefix {}...", clusterName, indexPrefix); - client = new CustomRestHighLevelClient(clientBuilder); + buildBulkIngester(); + LOGGER.info("Connecting to ElasticSearch persistence backend using index prefix {}...", indexPrefix); } - public BulkProcessor getBulkProcessor() { - if (bulkProcessor != null) { - return bulkProcessor; + public BulkIngester buildBulkIngester() { + if (bulkIngester != null) { + return bulkIngester; } - BulkProcessor.Listener bulkProcessorListener = new BulkProcessor.Listener() { - @Override - public void beforeBulk(long executionId, - BulkRequest request) { + BulkListener listener = new BulkListener() { + @Override public void beforeBulk(long executionId, BulkRequest request, List strings) { LOGGER.debug("Before Bulk"); } - @Override - public void afterBulk(long executionId, - BulkRequest request, - BulkResponse response) { + @Override public void afterBulk(long executionId, BulkRequest request, List strings, BulkResponse response) { LOGGER.debug("After Bulk"); } - @Override - public void afterBulk(long executionId, - BulkRequest request, - Throwable failure) { + @Override public void afterBulk(long executionId, BulkRequest request, List strings, Throwable failure) { LOGGER.error("After Bulk (failure)", failure); + } }; - BulkProcessor.Builder bulkProcessorBuilder = BulkProcessor.builder( - (request, bulkListener) -> - client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), - bulkProcessorListener); + + BulkIngester.Builder ingesterBuilder = new BulkIngester.Builder().client(esClient).maxOperations(100) + .flushInterval(1, TimeUnit.SECONDS).listener(listener); if (bulkProcessorConcurrentRequests != null) { int concurrentRequests = Integer.parseInt(bulkProcessorConcurrentRequests); if (concurrentRequests > 1) { - bulkProcessorBuilder.setConcurrentRequests(concurrentRequests); + ingesterBuilder.maxConcurrentRequests(concurrentRequests); } } if (bulkProcessorBulkActions != null) { int bulkActions = Integer.parseInt(bulkProcessorBulkActions); - bulkProcessorBuilder.setBulkActions(bulkActions); + ingesterBuilder.maxOperations(bulkActions); } if (bulkProcessorBulkSize != null) { - bulkProcessorBuilder.setBulkSize(ByteSizeValue.parseBytesSizeValue(bulkProcessorBulkSize, new ByteSizeValue(5, ByteSizeUnit.MB), BULK_PROCESSOR_BULK_SIZE)); + // Default is 5MB + ingesterBuilder.maxSize(bulkProcessorBulkSize * 1024 * 1024); } - if (bulkProcessorFlushInterval != null) { - bulkProcessorBuilder.setFlushInterval(TimeValue.parseTimeValue(bulkProcessorFlushInterval, null, BULK_PROCESSOR_FLUSH_INTERVAL)); + + if (bulkProcessorFlushIntervalInSeconds != null) { + ingesterBuilder.flushInterval(bulkProcessorFlushIntervalInSeconds, TimeUnit.SECONDS); } else { // in ElasticSearch this defaults to null, but we would like to set a value to 5 seconds by default - bulkProcessorBuilder.setFlushInterval(new TimeValue(5, TimeUnit.SECONDS)); + ingesterBuilder.flushInterval(5, TimeUnit.SECONDS); } if (bulkProcessorBackoffPolicy != null) { String backoffPolicyStr = bulkProcessorBackoffPolicy; if (backoffPolicyStr != null && backoffPolicyStr.length() > 0) { backoffPolicyStr = backoffPolicyStr.toLowerCase(); if ("nobackoff".equals(backoffPolicyStr)) { - bulkProcessorBuilder.setBackoffPolicy(BackoffPolicy.noBackoff()); + ingesterBuilder.backoffPolicy(BackoffPolicy.noBackoff()); } else if (backoffPolicyStr.startsWith("constant(")) { int paramStartPos = backoffPolicyStr.indexOf("constant(" + "constant(".length()); int paramEndPos = backoffPolicyStr.indexOf(")", paramStartPos); int paramSeparatorPos = backoffPolicyStr.indexOf(",", paramStartPos); - TimeValue delay = TimeValue.parseTimeValue(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos), new TimeValue(5, TimeUnit.SECONDS), BULK_PROCESSOR_BACKOFF_POLICY); + Long delay = Long.valueOf(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos)); + int maxNumberOfRetries = Integer.parseInt(backoffPolicyStr.substring(paramSeparatorPos + 1, paramEndPos)); - bulkProcessorBuilder.setBackoffPolicy(BackoffPolicy.constantBackoff(delay, maxNumberOfRetries)); + // Delay is in ms + ingesterBuilder.backoffPolicy(BackoffPolicy.constantBackoff(delay != null ? delay : 5000, maxNumberOfRetries)); } else if (backoffPolicyStr.startsWith("exponential")) { if (!backoffPolicyStr.contains("(")) { - bulkProcessorBuilder.setBackoffPolicy(BackoffPolicy.exponentialBackoff()); + ingesterBuilder.backoffPolicy(BackoffPolicy.exponentialBackoff()); } else { // we detected parameters, must process them. int paramStartPos = backoffPolicyStr.indexOf("exponential(" + "exponential(".length()); int paramEndPos = backoffPolicyStr.indexOf(")", paramStartPos); int paramSeparatorPos = backoffPolicyStr.indexOf(",", paramStartPos); - TimeValue delay = TimeValue.parseTimeValue(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos), new TimeValue(5, TimeUnit.SECONDS), BULK_PROCESSOR_BACKOFF_POLICY); + Long delay = Long.valueOf(backoffPolicyStr.substring(paramStartPos, paramSeparatorPos)); int maxNumberOfRetries = Integer.parseInt(backoffPolicyStr.substring(paramSeparatorPos + 1, paramEndPos)); - bulkProcessorBuilder.setBackoffPolicy(BackoffPolicy.exponentialBackoff(delay, maxNumberOfRetries)); + ingesterBuilder.backoffPolicy(BackoffPolicy.exponentialBackoff(delay != null ? delay : 5000, maxNumberOfRetries)); } } } } - bulkProcessor = bulkProcessorBuilder.build(); - return bulkProcessor; + bulkIngester = ingesterBuilder.build(); + return bulkIngester; } public void stop() { - - new InClassLoaderExecute(null, null, this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + new InClassLoaderExecute<>(null, null, this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { protected Object execute(Object... args) throws IOException { LOGGER.info("Closing ElasticSearch persistence backend..."); - if (bulkProcessor != null) { - try { - bulkProcessor.awaitClose(2, TimeUnit.MINUTES); - } catch (InterruptedException e) { - LOGGER.error("Error waiting for bulk operations to flush !", e); - } - } - if (client != null) { - client.close(); + if (esClient != null) { + esClient.close(); } return null; } @@ -687,7 +613,8 @@ protected Object execute(Object... args) throws IOException { public void bindConditionEvaluator(ServiceReference conditionEvaluatorServiceReference) { ConditionEvaluator conditionEvaluator = bundleContext.getService(conditionEvaluatorServiceReference); - conditionEvaluatorDispatcher.addEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString(), conditionEvaluator); + conditionEvaluatorDispatcher.addEvaluator(conditionEvaluatorServiceReference.getProperty("conditionEvaluatorId").toString(), + conditionEvaluator); } public void unbindConditionEvaluator(ServiceReference conditionEvaluatorServiceReference) { @@ -699,18 +626,19 @@ public void unbindConditionEvaluator(ServiceReference condit public void bindConditionESQueryBuilder(ServiceReference conditionESQueryBuilderServiceReference) { ConditionESQueryBuilder conditionESQueryBuilder = bundleContext.getService(conditionESQueryBuilderServiceReference); - conditionESQueryBuilderDispatcher.addQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString(), conditionESQueryBuilder); + conditionESQueryBuilderDispatcher.addQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString(), + conditionESQueryBuilder); } public void unbindConditionESQueryBuilder(ServiceReference conditionESQueryBuilderServiceReference) { if (conditionESQueryBuilderServiceReference == null) { return; } - conditionESQueryBuilderDispatcher.removeQueryBuilder(conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString()); + conditionESQueryBuilderDispatcher.removeQueryBuilder( + conditionESQueryBuilderServiceReference.getProperty("queryBuilderId").toString()); } - @Override - public void bundleChanged(BundleEvent event) { + @Override public void bundleChanged(BundleEvent event) { switch (event.getType()) { case BundleEvent.STARTING: loadPredefinedMappings(event.getBundle().getBundleContext(), true); @@ -719,39 +647,6 @@ public void bundleChanged(BundleEvent event) { } } - @Override - public void updated(Dictionary properties) { - Map propertyMappings = new HashMap<>(); - - // Boolean properties - propertyMappings.put("throwExceptions", ConfigurationUpdateHelper.booleanProperty(this::setThrowExceptions)); - propertyMappings.put("alwaysOverwrite", ConfigurationUpdateHelper.booleanProperty(this::setAlwaysOverwrite)); - propertyMappings.put("useBatchingForSave", ConfigurationUpdateHelper.booleanProperty(this::setUseBatchingForSave)); - propertyMappings.put("useBatchingForUpdate", ConfigurationUpdateHelper.booleanProperty(this::setUseBatchingForUpdate)); - propertyMappings.put("aggQueryThrowOnMissingDocs", ConfigurationUpdateHelper.booleanProperty(this::setAggQueryThrowOnMissingDocs)); - - // String properties - propertyMappings.put("logLevelRestClient", ConfigurationUpdateHelper.stringProperty(this::setLogLevelRestClient)); - propertyMappings.put("clientSocketTimeout", ConfigurationUpdateHelper.stringProperty(this::setClientSocketTimeout)); - propertyMappings.put("taskWaitingTimeout", ConfigurationUpdateHelper.stringProperty(this::setTaskWaitingTimeout)); - propertyMappings.put("taskWaitingPollingInterval", ConfigurationUpdateHelper.stringProperty(this::setTaskWaitingPollingInterval)); - propertyMappings.put("aggQueryMaxResponseSizeHttp", ConfigurationUpdateHelper.stringProperty(this::setAggQueryMaxResponseSizeHttp)); - - // Integer properties - propertyMappings.put("aggregateQueryBucketSize", ConfigurationUpdateHelper.integerProperty(this::setAggregateQueryBucketSize)); - - // Custom property for itemTypeToRefreshPolicy with IOException handling - propertyMappings.put("itemTypeToRefreshPolicy", ConfigurationUpdateHelper.customProperty((value, logger) -> { - try { - setItemTypeToRefreshPolicy(value.toString()); - } catch (IOException e) { - logger.warn("Error setting itemTypeToRefreshPolicy: {}", e.getMessage()); - } - })); - - ConfigurationUpdateHelper.processConfigurationUpdates(properties, LOGGER, "ElasticSearch persistence", propertyMappings); - } - private void loadPredefinedMappings(BundleContext bundleContext, boolean forceUpdateMapping) { Enumeration predefinedMappings = bundleContext.getBundle().findEntries("META-INF/cxs/mappings", "*.json", true); if (predefinedMappings == null) { @@ -814,26 +709,27 @@ private String loadMappingFile(URL predefinedMappingURL) throws IOException { return content.toString(); } - @Override - public List getAllItems(final Class clazz) { + @Override public String getName() { + return "elasticsearch"; + } + + @Override public List getAllItems(final Class clazz) { return getAllItems(clazz, 0, -1, null).getList(); } - @Override - public long getAllItemsCount(String itemType) { - return queryCount(QueryBuilders.matchAllQuery(), itemType); + @Override public long getAllItemsCount(String itemType) { + return queryCount(Query.of(q -> q.matchAll(m -> m)), itemType); } - @Override - public PartialList getAllItems(final Class clazz, int offset, int size, String sortBy) { + @Override public PartialList getAllItems(final Class clazz, int offset, int size, String sortBy) { return getAllItems(clazz, offset, size, sortBy, null); } - @Override - public PartialList getAllItems(final Class clazz, int offset, int size, String sortBy, String scrollTimeValidity) { + @Override public PartialList getAllItems(final Class clazz, int offset, int size, String sortBy, + String scrollTimeValidity) { long startTime = System.currentTimeMillis(); try { - return query(QueryBuilders.matchAllQuery(), sortBy, clazz, offset, size, null, scrollTimeValidity); + return query(Query.of(q -> q.matchAll(m -> m)), sortBy, clazz, offset, size, null, scrollTimeValidity); } finally { if (metricsService != null && metricsService.isActivated()) { metricsService.updateTimer(this.getClass().getName() + ".getAllItems", startTime); @@ -841,25 +737,19 @@ public PartialList getAllItems(final Class clazz, int off } } - @Override - public T load(final String itemId, final Class clazz) { + @Override public T load(final String itemId, final Class clazz) { return load(itemId, clazz, null); } - @Override - @Deprecated - public T load(final String itemId, final Date dateHint, final Class clazz) { + @Override @Deprecated public T load(final String itemId, final Date dateHint, final Class clazz) { return load(itemId, clazz, null); } - @Override - @Deprecated - public CustomItem loadCustomItem(final String itemId, final Date dateHint, String customItemType) { + @Override @Deprecated public CustomItem loadCustomItem(final String itemId, final Date dateHint, String customItemType) { return load(itemId, CustomItem.class, customItemType); } - @Override - public CustomItem loadCustomItem(final String itemId, String customItemType) { + @Override public CustomItem loadCustomItem(final String itemId, String customItemType) { return load(itemId, CustomItem.class, customItemType); } @@ -868,27 +758,25 @@ private T load(final String itemId, final Class clazz, final return null; } - return new InClassLoaderExecute(metricsService, this.getClass().getName() + ".loadItem", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + return new InClassLoaderExecute(metricsService, this.getClass().getName() + ".loadItem", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected T execute(Object... args) throws Exception { try { - String itemType = Item.getItemType(clazz); - if (customItemType != null) { - itemType = customItemType; - } + final String itemType = customItemType != null ? customItemType : Item.getItemType(clazz); String documentId = getDocumentIDForItemType(itemId, itemType); - boolean sessionSpecialDirectAccess = sessionLatestIndex != null && Session.ITEM_TYPE.equals(itemType) ; + boolean sessionSpecialDirectAccess = sessionLatestIndex != null && Session.ITEM_TYPE.equals(itemType); if (!sessionSpecialDirectAccess && isItemTypeRollingOver(itemType)) { return new MetricAdapter(metricsService, ".loadItemWithQuery") { - @Override - public T execute(Object... args) throws Exception { + @Override public T execute(Object... args) throws Exception { + Query query = Query.of(q -> q.ids(builder -> builder.values(documentId))); if (customItemType == null) { - PartialList r = query(QueryBuilders.idsQuery().addIds(documentId), null, clazz, 0, 1, null, null); + PartialList r = query(query, null, clazz, 0, 1, null, null); if (r.size() > 0) { return r.get(0); } } else { - PartialList r = query(QueryBuilders.idsQuery().addIds(documentId), null, customItemType, 0, 1, null, null); + PartialList r = query(query, null, customItemType, 0, 1, null, null); if (r.size() > 0) { return (T) r.get(0); } @@ -898,28 +786,29 @@ public T execute(Object... args) throws Exception { }.execute(); } else { // Special handling for session we check the latest available index directly to speed up session loading - GetRequest getRequest = new GetRequest(sessionSpecialDirectAccess ? sessionLatestIndex : getIndex(itemType), documentId); - GetResponse response = client.get(getRequest, RequestOptions.DEFAULT); - if (response.isExists()) { - String sourceAsString = response.getSourceAsString(); - final T value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz); - setMetadata(value, response.getId(), response.getVersion(), response.getSeqNo(), response.getPrimaryTerm(), response.getIndex()); + GetRequest getRequest = GetRequest.of( + builder -> builder.index(sessionSpecialDirectAccess ? sessionLatestIndex : getIndex(itemType)) + .id(documentId)); + GetResponse response = esClient.get(getRequest, clazz); + if (response.found()) { + T value = response.source(); + setMetadata(value, response.id(), response.version() != null ? response.version() : 0L, + response.seqNo() != null ? response.seqNo() : 0L, + response.primaryTerm() != null ? response.primaryTerm() : 0L, response.index()); return value; } else { return null; } } - } catch (ElasticsearchStatusException ese) { - if (ese.status().equals(RestStatus.NOT_FOUND)) { - // this can happen if we are just testing the existence of the item, it is not always an error. + } catch (ElasticsearchException e) { + if (e.status() == 404 && e.getMessage() != null && e.getMessage().contains("index_not_found_exception")) { + // The index does not exist return null; } - throw new Exception("Error loading itemType=" + clazz.getName() + " customItemType=" + customItemType + " itemId=" + itemId, ese); - } catch (IndexNotFoundException e) { - // this can happen if we are just testing the existence of the item, it is not always an error. return null; } catch (Exception ex) { - throw new Exception("Error loading itemType=" + clazz.getName() + " customItemType=" + customItemType + " itemId=" + itemId, ex); + throw new Exception( + "Error loading itemType=" + clazz.getName() + " customItemType=" + customItemType + " itemId=" + itemId, ex); } } }.catchingExecuteInClassLoader(true); @@ -936,30 +825,26 @@ private void setMetadata(Item item, String itemId, long version, long seqNo, lon item.setSystemMetadata("index", index); } - @Override - public boolean isConsistent(Item item) { - return getRefreshPolicy(item.getItemType()) != WriteRequest.RefreshPolicy.NONE; + @Override public boolean isConsistent(Item item) { + return getRefreshPolicy(item.getItemType()) != Refresh.False; } - @Override - public boolean save(final Item item) { + @Override public boolean save(final Item item) { return save(item, useBatchingForSave, alwaysOverwrite); } - @Override - public boolean save(final Item item, final boolean useBatching) { + @Override public boolean save(final Item item, final boolean useBatching) { return save(item, useBatching, alwaysOverwrite); } - @Override - public boolean save(final Item item, final Boolean useBatchingOption, final Boolean alwaysOverwriteOption) { + @Override public boolean save(final Item item, final Boolean useBatchingOption, final Boolean alwaysOverwriteOption) { final boolean useBatching = useBatchingOption == null ? this.useBatchingForSave : useBatchingOption; final boolean alwaysOverwrite = alwaysOverwriteOption == null ? this.alwaysOverwrite : alwaysOverwriteOption; - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".saveItem", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".saveItem", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { try { - String source = ESCustomObjectMapper.getObjectMapper().writeValueAsString(item); String itemType = item.getItemType(); if (item instanceof CustomItem) { itemType = ((CustomItem) item).getCustomItemType(); @@ -967,48 +852,61 @@ protected Boolean execute(Object... args) throws Exception { String documentId = getDocumentIDForItemType(item.getItemId(), itemType); String index = item.getSystemMetadata("index") != null ? (String) item.getSystemMetadata("index") : getIndex(itemType); - IndexRequest indexRequest = new IndexRequest(index); - indexRequest.id(documentId); - indexRequest.source(source, XContentType.JSON); - + Long seqNo; + Long primaryTerm; + OpType opType = null; + String routing; if (!alwaysOverwrite) { - Long seqNo = (Long) item.getSystemMetadata(SEQ_NO); - Long primaryTerm = (Long) item.getSystemMetadata(PRIMARY_TERM); - - if (seqNo != null && primaryTerm != null) { - indexRequest.setIfSeqNo(seqNo); - indexRequest.setIfPrimaryTerm(primaryTerm); - } else { - indexRequest.opType(DocWriteRequest.OpType.CREATE); - } + seqNo = (Long) item.getSystemMetadata(SEQ_NO); + primaryTerm = (Long) item.getSystemMetadata(PRIMARY_TERM); + opType = seqNo == null && primaryTerm == null ? OpType.Create : null; + } else { + primaryTerm = null; + seqNo = null; } if (routingByType.containsKey(itemType)) { - indexRequest.routing(routingByType.get(itemType)); + routing = routingByType.get(itemType); + } else { + routing = null; } try { - if (bulkProcessor == null || !useBatching) { - indexRequest.setRefreshPolicy(getRefreshPolicy(itemType)); - IndexResponse response = client.index(indexRequest, RequestOptions.DEFAULT); - String responseIndex = response.getIndex(); - String itemId = response.getId(); - setMetadata(item, itemId, response.getVersion(), response.getSeqNo(), response.getPrimaryTerm(), responseIndex); + if (bulkIngester == null || !useBatching) { + IndexRequest.Builder indexRequestBuilder = new IndexRequest.Builder<>().index(index).id(documentId) + .document(item).ifSeqNo(seqNo).ifPrimaryTerm(primaryTerm).opType(opType).routing(routing) + .refresh(getRefreshPolicy(itemType)); + IndexResponse response = esClient.index(indexRequestBuilder.build()); + String responseIndex = response.index(); + String itemId = response.id(); + setMetadata(item, itemId, response.version(), response.seqNo() != null ? response.seqNo() : 0L, + response.primaryTerm() != null ? response.primaryTerm() : 0L, responseIndex); // Special handling for session, in case of new session we check that a rollover happen or not to update the latest available index - if (Session.ITEM_TYPE.equals(itemType) && - sessionLatestIndex != null && - response.getResult().equals(DocWriteResponse.Result.CREATED) && - !responseIndex.equals(sessionLatestIndex)) { + if (Session.ITEM_TYPE.equals(itemType) && sessionLatestIndex != null && response.result().equals(Result.Created) + && !responseIndex.equals(sessionLatestIndex)) { sessionLatestIndex = responseIndex; } } else { - bulkProcessor.add(indexRequest); + BulkOperation bulkOp; + if (opType == OpType.Create) { + bulkOp = BulkOperation.of(b -> b.create( + c -> c.index(index).id(documentId).document(item).ifSeqNo(seqNo).ifPrimaryTerm(primaryTerm) + .routing(routing))); + } else { + bulkOp = BulkOperation.of(b -> b.index( + i -> i.index(index).id(documentId).document(item).ifSeqNo(seqNo).ifPrimaryTerm(primaryTerm) + .routing(routing))); + } + bulkIngester.add(bulkOp); } logMetadataItemOperation("saved", item); - } catch (IndexNotFoundException e) { - LOGGER.error("Could not find index {}, could not register item type {} with id {} ", index, itemType, item.getItemId(), e); - return false; + } catch (ElasticsearchException e) { + if (e.status() == 404 && e.getMessage() != null && e.getMessage().contains("index_not_found_exception")) { + LOGGER.error("Could not find index {}, could not register item type {} with id {} ", index, itemType, + item.getItemId(), e); + return false; + } } return true; } catch (IOException e) { @@ -1016,106 +914,118 @@ protected Boolean execute(Object... args) throws Exception { } } }.catchingExecuteInClassLoader(true); - if (result == null) { - return false; - } else { - return result; - } + return Objects.requireNonNullElse(result, false); } - @Override - public boolean update(final Item item, final Date dateHint, final Class clazz, final String propertyName, final Object propertyValue) { + @Override public boolean update(final Item item, final Date dateHint, final Class clazz, final String propertyName, + final Object propertyValue) { return update(item, clazz, propertyName, propertyValue); } - @Override - public boolean update(final Item item, final Date dateHint, final Class clazz, final Map source) { + @Override public boolean update(final Item item, final Date dateHint, final Class clazz, final Map source) { return update(item, clazz, source); } - @Override - public boolean update(final Item item, final Date dateHint, final Class clazz, final Map source, final boolean alwaysOverwrite) { + @Override public boolean update(final Item item, final Date dateHint, final Class clazz, final Map source, + final boolean alwaysOverwrite) { return update(item, clazz, source, alwaysOverwrite); } - @Override - public boolean update(final Item item, final Class clazz, final String propertyName, final Object propertyValue) { + @Override public boolean update(final Item item, final Class clazz, final String propertyName, final Object propertyValue) { return update(item, clazz, Collections.singletonMap(propertyName, propertyValue), alwaysOverwrite); } - - @Override - public boolean update(final Item item, final Class clazz, final Map source) { + @Override public boolean update(final Item item, final Class clazz, final Map source) { return update(item, clazz, source, alwaysOverwrite); } @Override + //TODO type Class and Map public boolean update(final Item item, final Class clazz, final Map source, final boolean alwaysOverwrite) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".updateItem", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".updateItem", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { try { - UpdateRequest updateRequest = createUpdateRequest(clazz, item, source, alwaysOverwrite); + // On suppose que cette méthode retourne un UpdateRequest + UpdateRequest updateRequest = createUpdateRequest(clazz, item, source, alwaysOverwrite); - if (bulkProcessor == null || !useBatchingForUpdate) { - UpdateResponse response = client.update(updateRequest, RequestOptions.DEFAULT); - setMetadata(item, response.getId(), response.getVersion(), response.getSeqNo(), response.getPrimaryTerm(), response.getIndex()); + if (bulkIngester == null || !useBatchingForUpdate) { + UpdateResponse response = esClient.update(updateRequest, clazz); + setMetadata(item, response.id(), response.version(), response.seqNo() != null ? response.seqNo() : 0L, + response.primaryTerm() != null ? response.primaryTerm() : 0L, response.index()); } else { - bulkProcessor.add(updateRequest); + BulkOperation bulkOp = BulkOperation.of(builder -> builder.update( + u -> u.index(updateRequest.index()).id(updateRequest.id()).action(b -> b.doc(updateRequest.doc())) + .ifSeqNo(updateRequest.ifSeqNo()).ifPrimaryTerm(updateRequest.ifPrimaryTerm()) + .routing(updateRequest.routing()))); + bulkIngester.add(bulkOp); } logMetadataItemOperation("updated", item); return true; - } catch (IndexNotFoundException e) { - throw new Exception("No index found for itemType=" + clazz.getName() + "itemId=" + item.getItemId(), e); + } catch (ElasticsearchException e) { + if (e.getMessage().contains("index_not_found_exception")) { + throw new Exception("No index found for itemType=" + clazz.getName() + " itemId=" + item.getItemId(), e); + } + throw e; } } }.catchingExecuteInClassLoader(true); return Objects.requireNonNullElse(result, false); } - private UpdateRequest createUpdateRequest(Class clazz, Item item, Map source, boolean alwaysOverwrite) { + private UpdateRequest createUpdateRequest(Class clazz, Item item, Map source, + boolean alwaysOverwrite) { String itemType = Item.getItemType(clazz); String documentId = getDocumentIDForItemType(item.getItemId(), itemType); - String index = item.getSystemMetadata("index") != null ? (String) item.getSystemMetadata("index") : getIndex(itemType); + String index = item.getSystemMetadata("index") != null ? (String) item.getSystemMetadata("index") : getIndex(itemType); - UpdateRequest updateRequest = new UpdateRequest(index, documentId); - updateRequest.doc(source); + UpdateRequest.Builder builder = new UpdateRequest.Builder<>().index(index).id(documentId).doc(source); if (!alwaysOverwrite) { Long seqNo = (Long) item.getSystemMetadata(SEQ_NO); Long primaryTerm = (Long) item.getSystemMetadata(PRIMARY_TERM); if (seqNo != null && primaryTerm != null) { - updateRequest.setIfSeqNo(seqNo); - updateRequest.setIfPrimaryTerm(primaryTerm); + builder.ifSeqNo(seqNo); + builder.ifPrimaryTerm(primaryTerm); } } - return updateRequest; + + return builder.build(); } - @Override - public List update(final Map items, final Date dateHint, final Class clazz) { + @Override public List update(final Map items, final Date dateHint, final Class clazz) { if (items.isEmpty()) return new ArrayList<>(); - List result = new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".updateItems", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + List result = new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".updateItems", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { protected List execute(Object... args) throws Exception { long batchRequestStartTime = System.currentTimeMillis(); - BulkRequest bulkRequest = new BulkRequest(); + List operations = new ArrayList<>(); + items.forEach((item, source) -> { UpdateRequest updateRequest = createUpdateRequest(clazz, item, source, alwaysOverwrite); - bulkRequest.add(updateRequest); + BulkOperation bulkOp = BulkOperation.of(builder -> builder.update( + u -> u.index(updateRequest.index()).id(updateRequest.id()).action(b -> b.doc(updateRequest.doc())) + .ifSeqNo(updateRequest.ifSeqNo()).ifPrimaryTerm(updateRequest.ifPrimaryTerm()) + .routing(updateRequest.routing()))); + operations.add(bulkOp); }); - BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT); - LOGGER.debug("{} profiles updated with bulk segment in {}ms", bulkRequest.numberOfActions(), System.currentTimeMillis() - batchRequestStartTime); + BulkRequest bulkRequest = new BulkRequest.Builder().operations(operations).build(); + BulkResponse bulkResponse = esClient.bulk(bulkRequest); + LOGGER.debug("{} profiles updated with bulk segment in {}ms", bulkRequest.operations().size(), + System.currentTimeMillis() - batchRequestStartTime); List failedItemsIds = new ArrayList<>(); - if (bulkResponse.hasFailures()) { - Iterator iterator = bulkResponse.iterator(); - iterator.forEachRemaining(bulkItemResponse -> { - failedItemsIds.add(bulkItemResponse.getId()); + if (bulkResponse.items().stream().anyMatch(item -> item.error() != null)) { + bulkResponse.items().forEach(item -> { + if (item.error() != null) { + failedItemsIds.add(item.id()); + } }); } return failedItemsIds; @@ -1125,72 +1035,90 @@ protected List execute(Object... args) throws Exception { return result; } - @Override - public boolean updateWithQueryAndScript(final Date dateHint, final Class clazz, final String[] scripts, final Map[] scriptParams, final Condition[] conditions) { + @Override public boolean updateWithQueryAndScript(final Date dateHint, final Class clazz, final String[] scripts, + final Map[] scriptParams, final Condition[] conditions) { return updateWithQueryAndScript(clazz, scripts, scriptParams, conditions); } - @Override - public boolean updateWithQueryAndScript(final Class clazz, final String[] scripts, final Map[] scriptParams, final Condition[] conditions) { + @Override public boolean updateWithQueryAndScript(final Class clazz, final String[] scripts, + final Map[] scriptParams, final Condition[] conditions) { Script[] builtScripts = new Script[scripts.length]; for (int i = 0; i < scripts.length; i++) { - builtScripts[i] = new Script(ScriptType.INLINE, "painless", scripts[i], scriptParams[i]); + Map jsonDataParams = scriptParams[i].entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue()))); + int finalI = i; + builtScripts[i] = Script.of(s -> s.lang(ScriptLanguage.Painless) + .source(ScriptSource.of(scriptSourceBuilder -> scriptSourceBuilder.scriptString(scripts[finalI]))) + .params(jsonDataParams)); } - return updateWithQueryAndScript(new Class[]{clazz}, builtScripts, conditions, true); + return updateWithQueryAndScript(new Class[] { clazz }, builtScripts, conditions, true); } - @Override - public boolean updateWithQueryAndStoredScript(Date dateHint, Class clazz, String[] scripts, Map[] scriptParams, Condition[] conditions) { - return updateWithQueryAndStoredScript(new Class[]{clazz}, scripts, scriptParams, conditions, true); + @Override public boolean updateWithQueryAndStoredScript(Date dateHint, Class clazz, String[] scripts, + Map[] scriptParams, Condition[] conditions) { + return updateWithQueryAndStoredScript(new Class[] { clazz }, scripts, scriptParams, conditions, true); } - @Override - public boolean updateWithQueryAndStoredScript(Class clazz, String[] scripts, Map[] scriptParams, Condition[] conditions) { - return updateWithQueryAndStoredScript(new Class[]{clazz}, scripts, scriptParams, conditions, true); + @Override public boolean updateWithQueryAndStoredScript(Class clazz, String[] scripts, Map[] scriptParams, + Condition[] conditions) { + return updateWithQueryAndStoredScript(new Class[] { clazz }, scripts, scriptParams, conditions, true); } - @Override - public boolean updateWithQueryAndStoredScript(Class[] classes, String[] scripts, Map[] scriptParams, Condition[] conditions, boolean waitForComplete) { + @Override public boolean updateWithQueryAndStoredScript(Class[] classes, String[] scripts, Map[] scriptParams, + Condition[] conditions, boolean waitForComplete) { Script[] builtScripts = new Script[scripts.length]; for (int i = 0; i < scripts.length; i++) { - builtScripts[i] = new Script(ScriptType.STORED, null, scripts[i], scriptParams[i]); + int finalI = i; + Map jsonDataParams = scriptParams[i].entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue()))); + builtScripts[i] = Script.of(s -> s.id(scripts[finalI]).params(jsonDataParams)); } return updateWithQueryAndScript(classes, builtScripts, conditions, waitForComplete); } - private boolean updateWithQueryAndScript(final Class[] classes, final Script[] scripts, final Condition[] conditions, boolean waitForComplete) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".updateWithQueryAndScript", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + private boolean updateWithQueryAndScript(final Class[] classes, final Script[] scripts, final Condition[] conditions, + boolean waitForComplete) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".updateWithQueryAndScript", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { String[] itemTypes = Arrays.stream(classes).map(Item::getItemType).toArray(String[]::new); String[] indices = Arrays.stream(itemTypes).map(itemType -> getIndexNameForQuery(itemType)).toArray(String[]::new); try { for (int i = 0; i < scripts.length; i++) { - RefreshRequest refreshRequest = new RefreshRequest(indices); - client.indices().refresh(refreshRequest, RequestOptions.DEFAULT); - - QueryBuilder queryBuilder = conditionESQueryBuilderDispatcher.buildFilter(conditions[i]); - UpdateByQueryRequest updateByQueryRequest = new UpdateByQueryRequest(indices); - updateByQueryRequest.setConflicts("proceed"); - updateByQueryRequest.setMaxRetries(1000); - updateByQueryRequest.setSlices(2); - updateByQueryRequest.setScript(scripts[i]); - updateByQueryRequest.setQuery(wrapWithItemsTypeQuery(itemTypes, queryBuilder)); - - TaskSubmissionResponse taskResponse = client.submitUpdateByQuery(updateByQueryRequest, RequestOptions.DEFAULT); - if (taskResponse == null) { - LOGGER.error("update with query and script: no response returned for query: {}", queryBuilder); + esClient.indices().refresh(r -> r.index(Arrays.asList(indices))); + + Query query = conditionESQueryBuilderDispatcher.buildFilter(conditions[i]); + int finalI = i; + + UpdateByQueryRequest updateByQueryRequest = UpdateByQueryRequest.of( + builder -> builder.index(List.of(indices)).conflicts(Conflicts.Proceed).waitForCompletion(false) + .slices(Slices.of(s -> s.value(2))).script(scripts[finalI]) + .query(wrapWithItemsTypeQuery(itemTypes, query))); + + UpdateByQueryResponse response = esClient.updateByQuery(updateByQueryRequest); + + if (response.task() == null) { + LOGGER.error("update with query and script: no response returned for query: {}", query); } else if (waitForComplete) { - waitForTaskComplete(updateByQueryRequest, taskResponse); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug( + "Waiting task [{}]: [{}] using query: [{}], polling every {}ms with a timeout configured to {}ms", + response.task(), updateByQueryRequest, updateByQueryRequest.query(), taskWaitingPollingInterval, + taskWaitingTimeout); + } + waitForTaskComplete(response.task()); } else { - LOGGER.debug("ES task started {}", taskResponse.getTask()); + LOGGER.debug("ES task started {}", response.task()); } } return true; - } catch (IndexNotFoundException e) { - throw new Exception("No index found for itemTypes=" + String.join(",", itemTypes), e); - } catch (ScriptException e) { - LOGGER.error("Error in the update script : {}\n{}\n{}", e.getScript(), e.getDetailedMessage(), e.getScriptStack()); + } catch (ElasticsearchException e) { + if (e.status() == 404 && e.getMessage() != null && e.getMessage().contains("index_not_found_exception")) { + throw new Exception("No index found for itemTypes=" + String.join(",", itemTypes), e); + } + //TODO check the message + LOGGER.error("Error in the update script : {}\n{}", e.response().toString(), e.getMessage(), e); throw new Exception("Error in the update script"); } } @@ -1202,32 +1130,28 @@ protected Boolean execute(Object... args) throws Exception { } } - private void waitForTaskComplete(AbstractBulkByScrollRequest request, TaskSubmissionResponse response) { - if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Waiting task [{}]: [{}] using query: [{}], polling every {}ms with a timeout configured to {}ms", - response.getTask(), request.toString(), request.getSearchRequest().source().query(), taskWaitingPollingInterval, taskWaitingTimeout); - } + private void waitForTaskComplete(String task) { long start = System.currentTimeMillis(); - new InClassLoaderExecute(metricsService, this.getClass().getName() + ".waitForTask", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + new InClassLoaderExecute(metricsService, this.getClass().getName() + ".waitForTask", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Void execute(Object... args) throws Exception { - TaskId taskId = new TaskId(response.getTask()); - while (true){ - Optional getTaskResponseOptional = client.tasks().get(new GetTaskRequest(taskId.getNodeId(), taskId.getId()), RequestOptions.DEFAULT); - if (getTaskResponseOptional.isPresent()) { - GetTaskResponse getTaskResponse = getTaskResponseOptional.get(); - if (getTaskResponse.isCompleted()) { + while (true) { + GetTasksResponse tasksResponse = esClient.tasks().get(GetTasksRequest.of(builder -> builder.taskId(task))); + if (tasksResponse != null) { + long taskId = tasksResponse.task().id(); + if (tasksResponse.completed()) { if (LOGGER.isDebugEnabled()) { - long millis = getTaskResponse.getTaskInfo().getRunningTimeNanos() / 1_000_000; + long millis = tasksResponse.task().runningTimeInNanos() / 1_000_000; long seconds = millis / 1000; - LOGGER.debug("Waiting task [{}]: Finished in {} {}", taskId, - seconds >= 1 ? seconds : millis, + LOGGER.debug("Waiting task [{}]: Finished in {} {}", taskId, seconds >= 1 ? seconds : millis, seconds >= 1 ? "seconds" : "milliseconds"); } break; } else { if ((start + taskWaitingTimeout) < System.currentTimeMillis()) { - LOGGER.error("Waiting task [{}]: Exceeded configured timeout ({}ms), aborting wait process", taskId, taskWaitingTimeout); + LOGGER.error("Waiting task [{}]: Exceeded configured timeout ({}ms), aborting wait process", taskId, + taskWaitingTimeout); break; } @@ -1239,7 +1163,7 @@ protected Void execute(Object... args) throws Exception { } } } else { - LOGGER.error("Waiting task [{}]: No task found", taskId); + LOGGER.error("Waiting task [{}]: No task found", task); break; } } @@ -1248,32 +1172,33 @@ protected Void execute(Object... args) throws Exception { }.catchingExecuteInClassLoader(true); } - @Override - public boolean storeScripts(Map scripts) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".storeScripts", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public boolean storeScripts(Map scripts) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".storeScripts", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { boolean executedSuccessfully = true; + for (Map.Entry script : scripts.entrySet()) { - PutStoredScriptRequest putStoredScriptRequest = new PutStoredScriptRequest(); - XContentBuilder builder = XContentFactory.jsonBuilder(); - builder.startObject(); - { - builder.startObject("script"); - { - builder.field("lang", "painless"); - builder.field("source", script.getValue()); + try { + // Construire la requête avec le nouveau client + PutScriptRequest putScriptRequest = PutScriptRequest.of(p -> p.id(script.getKey()).script(StoredScript.of( + s -> s.lang("painless").source(ScriptSource.of(builder -> builder.scriptString(script.getValue())))))); + + // Exécuter la requête + PutScriptResponse response = esClient.putScript(putScriptRequest); + + // Vérifier le résultat + boolean acknowledged = response.acknowledged(); + executedSuccessfully &= acknowledged; + + if (acknowledged) { + LOGGER.info("Successfully stored painless script: {}", script.getKey()); + } else { + LOGGER.error("Failed to store painless script: {}", script.getKey()); } - builder.endObject(); - } - builder.endObject(); - putStoredScriptRequest.content(BytesReference.bytes(builder), XContentType.JSON); - putStoredScriptRequest.id(script.getKey()); - AcknowledgedResponse response = client.putScript(putStoredScriptRequest, RequestOptions.DEFAULT); - executedSuccessfully &= response.isAcknowledged(); - if (response.isAcknowledged()) { - LOGGER.info("Successfully stored painless script: {}", script.getKey()); - } else { - LOGGER.error("Failed to store painless script: {}", script.getKey()); + } catch (Exception e) { + LOGGER.error("Exception while storing painless script: {}", script.getKey(), e); + executedSuccessfully = false; } } return executedSuccessfully; @@ -1282,59 +1207,69 @@ protected Boolean execute(Object... args) throws Exception { return Objects.requireNonNullElse(result, false); } - public boolean updateWithScript(final Item item, final Date dateHint, final Class clazz, final String script, final Map scriptParams) { + public boolean updateWithScript(final Item item, final Date dateHint, final Class clazz, final String script, + final Map scriptParams) { return updateWithScript(item, clazz, script, scriptParams); } - @Override - public boolean updateWithScript(final Item item, final Class clazz, final String script, final Map scriptParams) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".updateWithScript", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public boolean updateWithScript(final Item item, final Class clazz, final String script, + final Map scriptParams) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".updateWithScript", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { try { String itemType = Item.getItemType(clazz); String index = getIndex(itemType); String documentId = getDocumentIDForItemType(item.getItemId(), itemType); - Script actualScript = new Script(ScriptType.INLINE, "painless", script, scriptParams); + Map jsonDataParams = scriptParams.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> JsonData.of(entry.getValue()))); - UpdateRequest updateRequest = new UpdateRequest(index, documentId); + Script actualScript = Script.of(s -> s.lang(ScriptLanguage.Painless) + .source(ScriptSource.of(scriptSourceBuilder -> scriptSourceBuilder.scriptString(script))) + .params(jsonDataParams)); - Long seqNo = (Long) item.getSystemMetadata(SEQ_NO); - Long primaryTerm = (Long) item.getSystemMetadata(PRIMARY_TERM); + if (bulkIngester != null) { + UpdateOperation.Builder updateOperation = new UpdateOperation.Builder<>().index(index).id(documentId) + .ifSeqNo((Long) item.getSystemMetadata(SEQ_NO)).ifPrimaryTerm((Long) item.getSystemMetadata(PRIMARY_TERM)) + .action(UpdateAction.of(action -> action.script(actualScript))); + + BulkOperation operation = BulkOperation.of(op -> op.update(updateOperation.build())); + bulkIngester.add(operation); - if (seqNo != null && primaryTerm != null) { - updateRequest.setIfSeqNo(seqNo); - updateRequest.setIfPrimaryTerm(primaryTerm); - } - updateRequest.script(actualScript); - if (bulkProcessor == null) { - UpdateResponse response = client.update(updateRequest, RequestOptions.DEFAULT); - setMetadata(item, response.getId(), response.getVersion(), response.getSeqNo(), response.getPrimaryTerm(), response.getIndex()); } else { - bulkProcessor.add(updateRequest); + + UpdateRequest updateRequest = new UpdateRequest.Builder<>().index(index).id(documentId) + .ifSeqNo((Long) item.getSystemMetadata(SEQ_NO)).ifPrimaryTerm((Long) item.getSystemMetadata(PRIMARY_TERM)) + .script(actualScript).build(); + + UpdateResponse response = esClient.update(updateRequest, clazz); + setMetadata(item, response.id(), response.version(), response.seqNo(), response.primaryTerm(), response.index()); } return true; - } catch (IndexNotFoundException e) { - throw new Exception("No index found for itemType=" + clazz.getName() + "itemId=" + item.getItemId(), e); + } catch (ElasticsearchException e) { + if (e.status() == 404 && e.getMessage() != null && e.getMessage().contains("index_not_found_exception")) { + throw new Exception("No index found for itemType=" + clazz.getName() + "itemId=" + item.getItemId(), e); + } + throw new Exception("Error during update with script", e); } } }.catchingExecuteInClassLoader(true); return Objects.requireNonNullElse(result, false); } - @Override - public boolean remove(final String itemId, final Class clazz) { + @Override public boolean remove(final String itemId, final Class clazz) { return remove(itemId, clazz, null); } - @Override - public boolean removeCustomItem(final String itemId, final String customItemType) { + @Override public boolean removeCustomItem(final String itemId, final String customItemType) { return remove(itemId, CustomItem.class, customItemType); } private boolean remove(final String itemId, final Class clazz, String customItemType) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeItem", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeItem", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { try { String itemType = Item.getItemType(clazz); @@ -1344,10 +1279,10 @@ protected Boolean execute(Object... args) throws Exception { String documentId = getDocumentIDForItemType(itemId, itemType); String index = getIndexNameForQuery(itemType); - DeleteRequest deleteRequest = new DeleteRequest(index, documentId); - client.delete(deleteRequest, RequestOptions.DEFAULT); + esClient.delete(DeleteRequest.of(builder -> builder.index(index).id(documentId))); if (MetadataItem.class.isAssignableFrom(clazz)) { - LOGGER.info("Item of type {} with ID {} has been removed", customItemType != null ? customItemType : clazz.getSimpleName(), itemId); + LOGGER.info("Item of type {} with ID {} has been removed", + customItemType != null ? customItemType : clazz.getSimpleName(), itemId); } return true; } catch (Exception e) { @@ -1359,42 +1294,39 @@ protected Boolean execute(Object... args) throws Exception { } public boolean removeByQuery(final Condition query, final Class clazz) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeByQuery", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeByQuery", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { - QueryBuilder queryBuilder = conditionESQueryBuilderDispatcher.getQueryBuilder(query); - return removeByQuery(queryBuilder, clazz); + Query esQuery = conditionESQueryBuilderDispatcher.getQueryBuilder(query); + return removeByQuery(esQuery, clazz); } }.catchingExecuteInClassLoader(true); return Objects.requireNonNullElse(result, false); } - public boolean removeByQuery(QueryBuilder queryBuilder, final Class clazz) throws Exception { + public boolean removeByQuery(Query query, final Class clazz) throws Exception { try { String itemType = Item.getItemType(clazz); LOGGER.debug("Remove item of type {} using a query", itemType); - final DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest(getIndexNameForQuery(itemType)) - .setQuery(wrapWithItemTypeQuery(itemType, queryBuilder)) - // Setting slices to auto will let Elasticsearch choose the number of slices to use. - // This setting will use one slice per shard, up to a certain limit. - // The delete request will be more efficient and faster than no slicing. - .setSlices(AbstractBulkByScrollRequest.AUTO_SLICES) - // Elasticsearch takes a snapshot of the index when you hit delete by query request and uses the _version of the documents to process the request. - // If a document gets updated in the meantime, it will result in a version conflict error and the delete operation will fail. - // So we explicitly set the conflict strategy to proceed in case of version conflict. - .setAbortOnVersionConflict(false) - // Remove by Query is mostly used for purge and cleaning up old data - // It's mostly used in jobs/timed tasks so we don't really care about long request - // So we increase default timeout of 1min to 10min - .setTimeout(TimeValue.timeValueMinutes(removeByQueryTimeoutInMinutes)); - - TaskSubmissionResponse taskResponse = client.submitDeleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT); - - if (taskResponse == null) { - LOGGER.error("Remove by query: no response returned for query: {}", queryBuilder); + DeleteByQueryRequest deleteByQueryRequest = DeleteByQueryRequest.of( + builder -> builder.index(getIndexNameForQuery(itemType)).conflicts(Conflicts.Proceed) + .query(wrapWithItemTypeQuery(itemType, query)) + .timeout(Time.of(t -> t.time(removeByQueryTimeoutInMinutes + "m"))).waitForCompletion(false)); + + DeleteByQueryResponse deleteByQueryResponse = esClient.deleteByQuery(deleteByQueryRequest); + + String task = deleteByQueryResponse.task(); + if (task == null) { + LOGGER.error("Remove by query: no response returned for query: {}", query); return false; } - waitForTaskComplete(deleteByQueryRequest, taskResponse); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Waiting task [{}]: [{}] using query: [{}], polling every {}ms with a timeout configured to {}ms", task, + deleteByQueryRequest, deleteByQueryRequest.query(), taskWaitingPollingInterval, taskWaitingTimeout); + } + + waitForTaskComplete(task); return true; } catch (Exception e) { @@ -1403,61 +1335,63 @@ public boolean removeByQuery(QueryBuilder queryBuilder, final C } public boolean indexTemplateExists(final String templateName) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".indexTemplateExists", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".indexTemplateExists", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws IOException { - IndexTemplatesExistRequest indexTemplatesExistRequest = new IndexTemplatesExistRequest(templateName); - return client.indices().existsTemplate(indexTemplatesExistRequest, RequestOptions.DEFAULT); + return esClient.indices().existsIndexTemplate(ExistsIndexTemplateRequest.of(builder -> builder.name(templateName))).value(); } }.catchingExecuteInClassLoader(true); return Objects.requireNonNullElse(result, false); } public boolean removeIndexTemplate(final String templateName) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeIndexTemplate", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeIndexTemplate", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws IOException { - DeleteIndexTemplateRequest deleteIndexTemplateRequest = new DeleteIndexTemplateRequest(templateName); - AcknowledgedResponse deleteIndexTemplateResponse = client.indices().deleteTemplate(deleteIndexTemplateRequest, RequestOptions.DEFAULT); - return deleteIndexTemplateResponse.isAcknowledged(); + DeleteIndexTemplateRequest deleteIndexTemplateRequest = DeleteIndexTemplateRequest.of( + builder -> builder.name(templateName)); + return esClient.indices().deleteIndexTemplate(deleteIndexTemplateRequest).acknowledged(); } }.catchingExecuteInClassLoader(true); return Objects.requireNonNullElse(result, false); } - public boolean registerRolloverLifecyclePolicy() { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".createMonthlyIndexLifecyclePolicy", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + public void registerRolloverLifecyclePolicy() { + new InClassLoaderExecute(metricsService, this.getClass().getName() + ".createLifecyclePolicy", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws IOException { - // Create the lifecycle policy for monthly indices - Map phases = new HashMap<>(); - Map hotActions = new HashMap<>(); - final Long maxDocs = StringUtils.isEmpty(rolloverMaxDocs) ? null : Long.parseLong(rolloverMaxDocs); - hotActions.put( - RolloverAction.NAME, - new RolloverAction( - StringUtils.isEmpty(rolloverMaxSize) ? null : ByteSizeValue.parseBytesSizeValue(rolloverMaxSize, "rollover.maxSize"), - StringUtils.isEmpty(rolloverMaxAge) ? null : TimeValue.parseTimeValue(rolloverMaxAge, null, "rollover.maxAge"), - maxDocs - ) - ); - phases.put("hot", new Phase("hot", TimeValue.ZERO, hotActions)); - - LifecyclePolicy policy = new LifecyclePolicy(indexPrefix + "-" + ROLLOVER_LIFECYCLE_NAME, phases); - PutLifecyclePolicyRequest request = new PutLifecyclePolicyRequest(policy); - org.elasticsearch.client.core.AcknowledgedResponse putLifecyclePolicy = client.indexLifecycle().putLifecyclePolicy(request, RequestOptions.DEFAULT); - return putLifecyclePolicy.isAcknowledged(); + + RolloverAction.Builder rolloverActionBuilder = new RolloverAction.Builder(); + if (StringUtils.isNotEmpty(rolloverMaxAge)) { + rolloverActionBuilder.maxAge(new Time.Builder().time(rolloverMaxAge).build()); + } + if (StringUtils.isNotEmpty(rolloverMaxSize)) { + rolloverActionBuilder.maxSize(rolloverMaxSize); + } + if (StringUtils.isNotEmpty(rolloverMaxDocs)) { + rolloverActionBuilder.maxDocs(Long.parseLong(rolloverMaxDocs)); + } + RolloverAction rolloverAction = rolloverActionBuilder.build(); + + Phase hotPhase = new Phase.Builder().actions(new Actions.Builder().rollover(rolloverAction).build()) + .minAge(new Time.Builder().time("0ms").build()).build(); + IlmPolicy ilmPolicy = new IlmPolicy.Builder().phases(new Phases.Builder().hot(hotPhase).build()).build(); + PutLifecycleRequest request = new PutLifecycleRequest.Builder().policy(ilmPolicy) + .name(indexPrefix + "-" + ROLLOVER_LIFECYCLE_NAME).build(); + PutLifecycleResponse response = esClient.ilm().putLifecycle(request); + return response.acknowledged(); } }.catchingExecuteInClassLoader(true); - return Objects.requireNonNullElse(result, false); } public boolean createIndex(final String itemType) { LOGGER.debug("Create index {}", itemType); - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".createIndex", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".createIndex", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws IOException { String index = getIndex(itemType); - GetIndexRequest getIndexRequest = new GetIndexRequest(index); - boolean indexExists = client.indices().exists(getIndexRequest, RequestOptions.DEFAULT); - - if (!indexExists) { + BooleanResponse indexExists = esClient.indices().exists(ExistsRequest.of(builder -> builder.index(index))); + if (!indexExists.value()) { if (isItemTypeRollingOver(itemType)) { internalCreateRolloverTemplate(itemType); internalCreateRolloverIndex(index); @@ -1465,8 +1399,7 @@ protected Boolean execute(Object... args) throws IOException { internalCreateIndex(index, mappings.get(itemType)); } } - - return !indexExists; + return !indexExists.value(); } }.catchingExecuteInClassLoader(true); return Objects.requireNonNullElse(result, false); @@ -1475,13 +1408,13 @@ protected Boolean execute(Object... args) throws IOException { public boolean removeIndex(final String itemType) { String index = getIndex(itemType); - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeIndex", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeIndex", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws IOException { - GetIndexRequest getIndexRequest = new GetIndexRequest(index); - boolean indexExists = client.indices().exists(getIndexRequest, RequestOptions.DEFAULT); + boolean indexExists = esClient.indices().existsIndexTemplate(ExistsIndexTemplateRequest.of(builder -> builder.name(index))) + .value(); if (indexExists) { - DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(index); - client.indices().delete(deleteIndexRequest, RequestOptions.DEFAULT); + esClient.indices().delete(DeleteIndexRequest.of(builder -> builder.index(index))); } return indexExists; } @@ -1490,76 +1423,74 @@ protected Boolean execute(Object... args) throws IOException { } private void internalCreateRolloverTemplate(String itemName) throws IOException { - String rolloverAlias = indexPrefix + "-" + itemName; - PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest(rolloverAlias + "-rollover-template") - .patterns(Collections.singletonList(getRolloverIndexForQuery(itemName))) - .order(1) - .settings("{\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : " + StringUtils.defaultIfEmpty(rolloverIndexNumberOfShards, monthlyIndexNumberOfShards) + ",\n" + - " \"number_of_replicas\" : " + StringUtils.defaultIfEmpty(rolloverIndexNumberOfReplicas, monthlyIndexNumberOfReplicas) + ",\n" + - " \"mapping.total_fields.limit\" : " + StringUtils.defaultIfEmpty(rolloverIndexMappingTotalFieldsLimit, monthlyIndexMappingTotalFieldsLimit) + ",\n" + - " \"max_docvalue_fields_search\" : " + StringUtils.defaultIfEmpty(rolloverIndexMaxDocValueFieldsSearch, monthlyIndexMaxDocValueFieldsSearch) + ",\n" + - " \"lifecycle.name\": \"" + (indexPrefix + "-" + ROLLOVER_LIFECYCLE_NAME) + "\",\n" + - " \"lifecycle.rollover_alias\": \"" + rolloverAlias + "\"" + - "" + - " },\n" + - " \"analysis\": {\n" + - " \"analyzer\": {\n" + - " \"folding\": {\n" + - " \"type\":\"custom\",\n" + - " \"tokenizer\": \"keyword\",\n" + - " \"filter\": [ \"lowercase\", \"asciifolding\" ]\n" + - " }\n" + - " }\n" + - " }\n" + - "}\n", XContentType.JSON); - if (mappings.get(itemName) == null) { + if (!mappings.containsKey(itemName)) { LOGGER.warn("Couldn't find mapping for item {}, won't create monthly index template", itemName); return; } - putIndexTemplateRequest.mapping(mappings.get(itemName), XContentType.JSON); - client.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT); + + String rolloverAlias = buildRolloverAlias(itemName); + IndexSettingsAnalysis analysis = buildAnalysis(); + IndexSettings indexSettings = buildIndexSettings(rolloverAlias, analysis); + IndexTemplateMapping templateMapping = buildTemplateMapping(itemName, indexSettings); + + PutIndexTemplateRequest request = PutIndexTemplateRequest.of(builder -> builder.name(rolloverAlias + "-rollover-template") + .indexPatterns(Collections.singletonList(getRolloverIndexForQuery(itemName))).template(templateMapping).priority(1L)); + + esClient.indices().putIndexTemplate(request); + } + + private String buildRolloverAlias(String itemName) { + return indexPrefix + "-" + itemName; + } + + private IndexSettingsAnalysis buildAnalysis() { + return IndexSettingsAnalysis.of(an -> an.analyzer("folding", analyserBuilder -> analyserBuilder.custom( + CustomAnalyzer.of(customAnalyzer -> customAnalyzer.tokenizer("keyword").filter("lowercase", "asciifolding"))))); + } + + private IndexSettings buildIndexSettings(String rolloverAlias, IndexSettingsAnalysis analysis) { + return IndexSettings.of(builder -> builder.index( + indexBuilder -> indexBuilder.numberOfShards(rolloverIndexNumberOfShards).numberOfReplicas(rolloverIndexNumberOfReplicas) + .mapping(MappingLimitSettings.of(limitBuilder -> limitBuilder.totalFields(MappingLimitSettingsTotalFields.of( + totalFieldLimitBuilder -> totalFieldLimitBuilder.limit(rolloverIndexMappingTotalFieldsLimit))))) + .maxDocvalueFieldsSearch(Integer.valueOf(rolloverIndexMaxDocValueFieldsSearch)).lifecycle( + lifecycleBuilder -> lifecycleBuilder.name(indexPrefix + "-" + ROLLOVER_LIFECYCLE_NAME) + .rolloverAlias(rolloverAlias))).analysis(analysis)); + } + + private IndexTemplateMapping buildTemplateMapping(String itemName, IndexSettings indexSettings) { + return IndexTemplateMapping.of(templateMappingBuilder -> templateMappingBuilder.settings(indexSettings).mappings( + mappingsBuilder -> mappingsBuilder.withJson( + new ByteArrayInputStream(mappings.get(itemName).getBytes(StandardCharsets.UTF_8))))); } private void internalCreateRolloverIndex(String indexName) throws IOException { - CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName + "-000001") - .alias(new Alias(indexName).writeIndex(true)); - CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT); + CreateIndexResponse createIndexResponse = esClient.indices().create(CreateIndexRequest.of( + builder -> builder.index(indexName + "-000001") + .aliases(indexName, Alias.of(aliasBuilder -> aliasBuilder.isWriteIndex(true))))); LOGGER.info("Index created: [{}], acknowledge: [{}], shards acknowledge: [{}]", createIndexResponse.index(), - createIndexResponse.isAcknowledged(), createIndexResponse.isShardsAcknowledged()); + createIndexResponse.acknowledged(), createIndexResponse.shardsAcknowledged()); } private void internalCreateIndex(String indexName, String mappingSource) throws IOException { - CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); - createIndexRequest.settings("{\n" + - " \"index\" : {\n" + - " \"number_of_shards\" : " + numberOfShards + ",\n" + - " \"number_of_replicas\" : " + numberOfReplicas + ",\n" + - " \"mapping.total_fields.limit\" : " + indexMappingTotalFieldsLimit + ",\n" + - " \"max_docvalue_fields_search\" : " + indexMaxDocValueFieldsSearch + "\n" + - " },\n" + - " \"analysis\": {\n" + - " \"analyzer\": {\n" + - " \"folding\": {\n" + - " \"type\":\"custom\",\n" + - " \"tokenizer\": \"keyword\",\n" + - " \"filter\": [ \"lowercase\", \"asciifolding\" ]\n" + - " }\n" + - " }\n" + - " }\n" + - "}\n", XContentType.JSON); + IndexSettings indexSettings = IndexSettings.of(builder -> builder.numberOfShards(numberOfShards).numberOfReplicas(numberOfReplicas) + .mapping(MappingLimitSettings.of(limitBuilder -> limitBuilder.totalFields(MappingLimitSettingsTotalFields.of( + totalFieldLimitBuilder -> totalFieldLimitBuilder.limit(indexMappingTotalFieldsLimit))))) + .maxDocvalueFieldsSearch(Integer.valueOf(indexMaxDocValueFieldsSearch)).analysis(buildAnalysis())); + CreateIndexRequest.Builder createIndexRequestBuilder = new CreateIndexRequest.Builder(); + createIndexRequestBuilder.index(indexName).settings(indexSettings); if (mappingSource != null) { - createIndexRequest.mapping(mappingSource, XContentType.JSON); + createIndexRequestBuilder.mappings( + mappingsBuilder -> mappingsBuilder.withJson(new ByteArrayInputStream(mappingSource.getBytes(StandardCharsets.UTF_8)))); } - CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT); + CreateIndexResponse createIndexResponse = esClient.indices().create(createIndexRequestBuilder.build()); + LOGGER.info("Index created: [{}], acknowledge: [{}], shards acknowledge: [{}]", createIndexResponse.index(), - createIndexResponse.isAcknowledged(), createIndexResponse.isShardsAcknowledged()); + createIndexResponse.acknowledged(), createIndexResponse.shardsAcknowledged()); } - @Override - public void createMapping(String type, String source) { + @Override public void createMapping(String type, String source) { try { putMapping(source, getIndex(type)); } catch (IOException ioe) { @@ -1658,48 +1589,52 @@ private String convertValueTypeToESType(String valueTypeId) { } } - private boolean putMapping(final String source, final String indexName) throws IOException { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".putMapping", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + private void putMapping(final String source, final String indexName) throws IOException { + new InClassLoaderExecute(metricsService, this.getClass().getName() + ".putMapping", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { try { - PutMappingRequest putMappingRequest = new PutMappingRequest(indexName); - putMappingRequest.source(source, XContentType.JSON); - AcknowledgedResponse response = client.indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); - return response.isAcknowledged(); + PutMappingResponse putMappingResponse = esClient.indices().putMapping(PutMappingRequest.of( + builder -> builder.index(indexName) + .withJson(new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))))); + return putMappingResponse.acknowledged(); } catch (Exception e) { throw new Exception("Cannot create/update mapping", e); } } }.catchingExecuteInClassLoader(true); - return Objects.requireNonNullElse(result, false); } - @Override - public Map> getPropertiesMapping(final String itemType) { - return new InClassLoaderExecute>>(metricsService, this.getClass().getName() + ".getPropertiesMapping", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - @SuppressWarnings("unchecked") - protected Map> execute(Object... args) throws Exception { + @Override public Map> getPropertiesMapping(final String itemType) { + return new InClassLoaderExecute>>(metricsService, + this.getClass().getName() + ".getPropertiesMapping", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @SuppressWarnings("unchecked") protected Map> execute(Object... args) throws Exception { // Get all mapping for current itemType - GetMappingsRequest getMappingsRequest = new GetMappingsRequest(); - getMappingsRequest.indices(getIndexNameForQuery(itemType)); - GetMappingsResponse getMappingsResponse = client.indices().getMapping(getMappingsRequest, RequestOptions.DEFAULT); - Map mappings = getMappingsResponse.mappings(); + GetMappingRequest getMappingsRequest = GetMappingRequest.of(r -> r.index(getIndexNameForQuery(itemType))); + GetMappingResponse getMappingsResponse = esClient.indices().getMapping(getMappingsRequest); + Map mappings = getMappingsResponse.mappings(); // create a list of Keys to get the mappings in chronological order - // in case there is monthly context then the mapping will be added from the oldest to the most recent one Set orderedKeys = new TreeSet<>(mappings.keySet()); Map> result = new HashMap<>(); try { for (String key : orderedKeys) { if (mappings.containsKey(key)) { - Map> properties = (Map>) mappings.get(key).getSourceAsMap().get("properties"); - for (Map.Entry> entry : properties.entrySet()) { + TypeMapping typeMapping = mappings.get(key).mappings(); + if (typeMapping == null || typeMapping.properties() == null) + continue; + Map properties = typeMapping.properties(); + // Convert Property to Map + Map> propertiesMap = new HashMap<>(); + for (Map.Entry entry : properties.entrySet()) { + propertiesMap.put(entry.getKey(), propertyToMap(entry.getValue())); + } + + for (Map.Entry> entry : propertiesMap.entrySet()) { if (result.containsKey(entry.getKey())) { Map subResult = result.get(entry.getKey()); - for (Map.Entry subentry : entry.getValue().entrySet()) { - if (subResult.containsKey(subentry.getKey()) - && subResult.get(subentry.getKey()) instanceof Map + if (subResult.containsKey(subentry.getKey()) && subResult.get(subentry.getKey()) instanceof Map && subentry.getValue() instanceof Map) { mergePropertiesMapping((Map) subResult.get(subentry.getKey()), (Map) subentry.getValue()); } else { @@ -1720,13 +1655,33 @@ protected Map> execute(Object... args) throws Except }.catchingExecuteInClassLoader(true); } + /** + * Converts a Property into a generic Map + * to maintain compatibility with the old code using getSourceAsMap(). + */ + @SuppressWarnings("unchecked") private Map propertyToMap(Property property) { + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + JsonpMapper mapper = esClient._transport().jsonpMapper(); + JsonGenerator generator = mapper.jsonProvider().createGenerator(baos); + mapper.serialize(property, generator); + generator.close(); + + String json = baos.toString(StandardCharsets.UTF_8); + ObjectMapper jackson = new ObjectMapper(); + return jackson.readValue(json, new TypeReference<>() { + }); + } catch (Exception e) { + return new HashMap<>(); + } + } + private void mergePropertiesMapping(Map result, Map entry) { if (entry == null || entry.isEmpty()) { return; } for (Map.Entry subentry : entry.entrySet()) { - if (result.containsKey(subentry.getKey()) - && result.get(subentry.getKey()) instanceof Map + if (result.containsKey(subentry.getKey()) && result.get(subentry.getKey()) instanceof Map && subentry.getValue() instanceof Map) { mergePropertiesMapping((Map) result.get(subentry.getKey()), (Map) subentry.getValue()); } else { @@ -1766,70 +1721,18 @@ private String getPropertyNameWithData(String name, String itemType) { if (propertyMapping == null) { return null; } - if ("text".equals(propertyMapping.get("type")) - && propertyMapping.containsKey("fields") - && ((Map) propertyMapping.get("fields")).containsKey("keyword")) { + if ("text".equals(propertyMapping.get("type")) && propertyMapping.containsKey("fields") && ((Map) propertyMapping.get( + "fields")).containsKey("keyword")) { name += ".keyword"; } return name; } - public boolean saveQuery(final String queryName, final String query) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".saveQuery", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - protected Boolean execute(Object... args) throws Exception { - //Index the query = register it in the percolator - try { - LOGGER.info("Saving query : {}", queryName); - String index = getIndex(".percolator"); - IndexRequest indexRequest = new IndexRequest(index); - indexRequest.id(queryName); - indexRequest.source(query, XContentType.JSON); - indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client.index(indexRequest, RequestOptions.DEFAULT); - return true; - } catch (Exception e) { - throw new Exception("Cannot save query", e); - } - } - }.catchingExecuteInClassLoader(true); - return Objects.requireNonNullElse(result, false); - } - - @Override - public boolean saveQuery(String queryName, Condition query) { - if (query == null) { - return false; - } - saveQuery(queryName, conditionESQueryBuilderDispatcher.getQuery(query)); - return true; - } - - @Override - public boolean removeQuery(final String queryName) { - Boolean result = new InClassLoaderExecute(metricsService, this.getClass().getName() + ".removeQuery", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - protected Boolean execute(Object... args) throws Exception { - //Index the query = register it in the percolator - try { - String index = getIndex(".percolator"); - DeleteRequest deleteRequest = new DeleteRequest(index); - deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - client.delete(deleteRequest, RequestOptions.DEFAULT); - return true; - } catch (Exception e) { - throw new Exception("Cannot delete query", e); - } - } - }.catchingExecuteInClassLoader(true); - return Objects.requireNonNullElse(result, false); - } - - @Override - public boolean isValidCondition(Condition condition, Item item) { + @Override public boolean isValidCondition(Condition condition, Item item) { try { conditionEvaluatorDispatcher.eval(condition, item); - QueryBuilders.boolQuery() - .must(QueryBuilders.idsQuery().addIds(item.getItemId())) - .must(conditionESQueryBuilderDispatcher.buildFilter(condition)); + Query.of(q -> q.bool(builder -> builder.must(mustBuilder -> mustBuilder.ids(IdsQuery.of(ids -> ids.values(item.getItemId())))) + .must(conditionESQueryBuilderDispatcher.buildFilter(condition)))); } catch (Exception e) { LOGGER.error("Failed to validate condition. See debug log level for more information"); LOGGER.debug("Failed to validate condition, condition={}", condition, e); @@ -1838,8 +1741,7 @@ public boolean isValidCondition(Condition condition, Item item) { return true; } - @Override - public boolean testMatch(Condition query, Item item) { + @Override public boolean testMatch(Condition query, Item item) { long startTime = System.currentTimeMillis(); try { return conditionEvaluatorDispatcher.eval(query, item); @@ -1856,10 +1758,10 @@ public boolean testMatch(Condition query, Item item) { String itemType = Item.getItemType(clazz); String documentId = getDocumentIDForItemType(item.getItemId(), itemType); - QueryBuilder builder = QueryBuilders.boolQuery() - .must(QueryBuilders.idsQuery().addIds(documentId)) - .must(conditionESQueryBuilderDispatcher.buildFilter(query)); - return queryCount(builder, itemType) > 0; + Query esQuery = Query.of(q -> q.bool( + builder -> builder.must(mustBuilder -> mustBuilder.ids(IdsQuery.of(ids -> ids.values(documentId)))) + .must(conditionESQueryBuilderDispatcher.buildFilter(query)))); + return queryCount(esQuery, itemType) > 0; } finally { if (metricsService != null && metricsService.isActivated()) { metricsService.updateTimer(this.getClass().getName() + ".testMatchInElasticSearch", startTime); @@ -1867,74 +1769,74 @@ public boolean testMatch(Condition query, Item item) { } } - - @Override - public List query(final Condition query, String sortBy, final Class clazz) { + @Override public List query(final Condition query, String sortBy, final Class clazz) { return query(query, sortBy, clazz, 0, -1).getList(); } - @Override - public PartialList query(final Condition query, String sortBy, final Class clazz, final int offset, final int size) { + @Override public PartialList query(final Condition query, String sortBy, final Class clazz, final int offset, + final int size) { return query(conditionESQueryBuilderDispatcher.getQueryBuilder(query), sortBy, clazz, offset, size, null, null); } - @Override - public PartialList query(final Condition query, String sortBy, final Class clazz, final int offset, final int size, final String scrollTimeValidity) { + @Override public PartialList query(final Condition query, String sortBy, final Class clazz, final int offset, + final int size, final String scrollTimeValidity) { return query(conditionESQueryBuilderDispatcher.getQueryBuilder(query), sortBy, clazz, offset, size, null, scrollTimeValidity); } - @Override - public PartialList queryCustomItem(final Condition query, String sortBy, final String customItemType, final int offset, final int size, final String scrollTimeValidity) { - return query(conditionESQueryBuilderDispatcher.getQueryBuilder(query), sortBy, customItemType, offset, size, null, scrollTimeValidity); + @Override public PartialList queryCustomItem(final Condition query, String sortBy, final String customItemType, + final int offset, final int size, final String scrollTimeValidity) { + return query(conditionESQueryBuilderDispatcher.getQueryBuilder(query), sortBy, customItemType, offset, size, null, + scrollTimeValidity); } - @Override - public PartialList queryFullText(final String fulltext, final Condition query, String sortBy, final Class clazz, final int offset, final int size) { - return query(QueryBuilders.boolQuery().must(QueryBuilders.queryStringQuery(fulltext)).must(conditionESQueryBuilderDispatcher.getQueryBuilder(query)), sortBy, clazz, offset, size, null, null); + @Override public PartialList queryFullText(final String fulltext, final Condition query, String sortBy, + final Class clazz, final int offset, final int size) { + return query(Query.of(builder -> builder.bool( + boolBuilder -> boolBuilder.must(mustBuilder -> mustBuilder.queryString(qsBuilder -> qsBuilder.query(fulltext))) + .filter(conditionESQueryBuilderDispatcher.getQueryBuilder(query)))), sortBy, clazz, offset, size, null, null); } - @Override - public List query(final String fieldName, final String fieldValue, String sortBy, final Class clazz) { + @Override public List query(final String fieldName, final String fieldValue, String sortBy, final Class clazz) { return query(fieldName, fieldValue, sortBy, clazz, 0, -1).getList(); } - @Override - public List query(final String fieldName, final String[] fieldValues, String sortBy, final Class clazz) { - return query(QueryBuilders.termsQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValues)), sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList(); + @Override public List query(final String fieldName, final String[] fieldValues, String sortBy, + final Class clazz) { + Query termQuery = Query.of(builder -> builder.terms(t -> t.field(fieldName).terms(TermsQueryField.of( + termsBuilder -> termsBuilder.value( + Arrays.stream(fieldValues).map(fieldValue -> FieldValue.of(ConditionContextHelper.foldToASCII(fieldValue))) + .toList()))))); + return query(termQuery, sortBy, clazz, 0, -1, getRouting(fieldName, fieldValues, clazz), null).getList(); } - @Override - public PartialList query(String fieldName, String fieldValue, String sortBy, Class clazz, int offset, int size) { - return query(termQuery(fieldName, ConditionContextHelper.foldToASCII(fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null); - } + @Override public PartialList query(String fieldName, String fieldValue, String sortBy, Class clazz, int offset, + int size) { - @Override - public PartialList queryFullText(String fieldName, String fieldValue, String fulltext, String sortBy, Class clazz, int offset, int size) { - return query(QueryBuilders.boolQuery().must(QueryBuilders.queryStringQuery(fulltext)).must(termQuery(fieldName, fieldValue)), sortBy, clazz, offset, size, getRouting(fieldName, new String[]{fieldValue}, clazz), null); + Query termQuery = Query.of(builder -> builder.terms(t -> t.field(fieldName).terms(TermsQueryField.of( + termsBuilder -> termsBuilder.value(List.of(FieldValue.of(ConditionContextHelper.foldToASCII(fieldValue)))))))); + return query(termQuery, sortBy, clazz, offset, size, getRouting(fieldName, new String[] { fieldValue }, clazz), null); } - @Override - public PartialList queryFullText(String fulltext, String sortBy, Class clazz, int offset, int size) { - return query(QueryBuilders.queryStringQuery(fulltext), sortBy, clazz, offset, size, null, null); + @Override public PartialList queryFullText(String fieldName, String fieldValue, String fulltext, String sortBy, + Class clazz, int offset, int size) { + Query query = Query.of(q -> q.bool(b -> b.must(Query.of(qs -> qs.queryString(qsq -> qsq.query(fulltext)))) + .must(Query.of(t -> t.term(term -> term.field(fieldName).value(fieldValue)))))); + return query(query, sortBy, clazz, offset, size, getRouting(fieldName, new String[] { fieldValue }, clazz), null); } - @Override - public PartialList rangeQuery(String fieldName, String from, String to, String sortBy, Class clazz, int offset, int size) { - RangeQueryBuilder builder = QueryBuilders.rangeQuery(fieldName); - builder.from(from); - builder.to(to); - return query(builder, sortBy, clazz, offset, size, null, null); + @Override public PartialList queryFullText(String fulltext, String sortBy, Class clazz, int offset, int size) { + return query(Query.of(q -> q.queryString(qs -> qs.query(fulltext))), sortBy, clazz, offset, size, null, null); } - @Override - public long queryCount(Condition query, String itemType) { + @Override public long queryCount(Condition query, String itemType) { try { return conditionESQueryBuilderDispatcher.count(query); } catch (UnsupportedOperationException e) { try { - QueryBuilder filter = conditionESQueryBuilderDispatcher.buildFilter(query); - if (filter instanceof IdsQueryBuilder) { - return ((IdsQueryBuilder) filter).ids().size(); + Query filter = conditionESQueryBuilderDispatcher.buildFilter(query); + + if (filter.isIds()) { + return filter.ids().values().size(); } return queryCount(filter, itemType); } catch (UnsupportedOperationException e1) { @@ -1943,137 +1845,121 @@ public long queryCount(Condition query, String itemType) { } } - private long queryCount(final QueryBuilder filter, final String itemType) { - return new InClassLoaderExecute(metricsService, this.getClass().getName() + ".queryCount", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - - @Override - protected Long execute(Object... args) throws IOException { + private long queryCount(final Query query, final String itemType) { + return new InClassLoaderExecute(metricsService, this.getClass().getName() + ".queryCount", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { - CountRequest countRequest = new CountRequest(getIndexNameForQuery(itemType)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.query(wrapWithItemTypeQuery(itemType, filter)); - countRequest.source(searchSourceBuilder); - CountResponse response = client.count(countRequest, RequestOptions.DEFAULT); - return response.getCount(); + @Override protected Long execute(Object... args) throws IOException { + CountRequest countRequest = CountRequest.of( + builder -> builder.index(getIndexNameForQuery(itemType)).query(wrapWithItemTypeQuery(itemType, query))); + return esClient.count(countRequest).count(); } }.catchingExecuteInClassLoader(true); } - private PartialList query(final QueryBuilder query, final String sortBy, final Class clazz, final int offset, final int size, final String[] routing, final String scrollTimeValidity) { + private PartialList query(final Query query, final String sortBy, final Class clazz, final int offset, + final int size, final String[] routing, final String scrollTimeValidity) { return query(query, sortBy, clazz, null, offset, size, routing, scrollTimeValidity); } - private PartialList query(final QueryBuilder query, final String sortBy, final String customItemType, final int offset, final int size, final String[] routing, final String scrollTimeValidity) { + private PartialList query(final Query query, final String sortBy, final String customItemType, final int offset, + final int size, final String[] routing, final String scrollTimeValidity) { return query(query, sortBy, CustomItem.class, customItemType, offset, size, routing, scrollTimeValidity); } - private PartialList query(final QueryBuilder query, final String sortBy, final Class clazz, final String customItemType, final int offset, final int size, final String[] routing, final String scrollTimeValidity) { - return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".query", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + private PartialList query(final Query query, final String sortBy, final Class clazz, final String customItemType, + final int offset, final int size, final String[] routing, final String scrollTimeValidity) { + return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".query", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { - @Override - protected PartialList execute(Object... args) throws Exception { - List results = new ArrayList(); + @Override protected PartialList execute(Object... args) throws Exception { + List results = new ArrayList<>(); String scrollIdentifier = null; long totalHits = 0; PartialList.Relation totalHitsRelation = PartialList.Relation.EQUAL; try { - String itemType = Item.getItemType(clazz); - if (customItemType != null) { - itemType = customItemType; - } - TimeValue keepAlive = TimeValue.timeValueHours(1); - SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - .fetchSource(true) - .seqNoAndPrimaryTerm(true) - .query(wrapWithItemTypeQuery(itemType, query)) - .size(size < 0 ? defaultQueryLimit : size) - .from(offset); + String itemType = customItemType != null ? customItemType : Item.getItemType(clazz); + int limit = size < 0 ? defaultQueryLimit : size; + + SearchRequest.Builder searchRequest = new SearchRequest.Builder(); + searchRequest.index(getIndexNameForQuery(itemType)).from(offset).size(limit) + .query(wrapWithItemTypeQuery(itemType, query)).seqNoPrimaryTerm(true).source(src -> src.fetch(true)); + + Time keepAlive = Time.of(t -> t.time("1h")); + if (scrollTimeValidity != null) { - keepAlive = TimeValue.parseTimeValue(scrollTimeValidity, TimeValue.timeValueHours(1), "scrollTimeValidity"); + keepAlive = Time.of(t -> t.time(scrollTimeValidity.isBlank() ? "1h" : scrollTimeValidity)); searchRequest.scroll(keepAlive); } if (size == Integer.MIN_VALUE) { - searchSourceBuilder.size(defaultQueryLimit); + searchRequest.size(defaultQueryLimit); } else if (size != -1) { - searchSourceBuilder.size(size); + searchRequest.size(size); } else { // size == -1, use scroll query to retrieve all the results searchRequest.scroll(keepAlive); } if (routing != null) { - searchRequest.routing(routing); + searchRequest.routing(String.join(",", routing)); } if (sortBy != null) { String[] sortByArray = sortBy.split(","); for (String sortByElement : sortByArray) { if (sortByElement.startsWith("geo:")) { String[] elements = sortByElement.split(":"); - GeoDistanceSortBuilder distanceSortBuilder = SortBuilders.geoDistanceSort(elements[1], Double.parseDouble(elements[2]), Double.parseDouble(elements[3])).unit(DistanceUnit.KILOMETERS); - if (elements.length > 4 && elements[4].equals("desc")) { - searchSourceBuilder.sort(distanceSortBuilder.order(SortOrder.DESC)); - } else { - searchSourceBuilder.sort(distanceSortBuilder.order(SortOrder.ASC)); - } + GeoLocation location = GeoLocation.of(g -> g.latlon( + latlon -> latlon.lat(Double.parseDouble(elements[2])).lon(Double.parseDouble(elements[3])))); + + SortOrder order = (elements.length > 4 && "desc".equals(elements[4])) ? SortOrder.Desc : SortOrder.Asc; + + GeoDistanceSort geoSort = GeoDistanceSort.of(g -> g.field(elements[1]).location(location) + .unit(co.elastic.clients.elasticsearch._types.DistanceUnit.Kilometers).order(order)); + searchRequest.sort(s -> s.geoDistance(geoSort)); } else { String name = getPropertyNameWithData(StringUtils.substringBeforeLast(sortByElement, ":"), itemType); if (name != null) { - if (sortByElement.endsWith(":desc")) { - searchSourceBuilder.sort(name, SortOrder.DESC); - } else { - searchSourceBuilder.sort(name, SortOrder.ASC); - } - } else { - // in the case of no data existing for the property, we will not add the sorting to the request. - } + SortOrder sortOrder = sortByElement.endsWith(":desc") ? SortOrder.Desc : SortOrder.Asc; + searchRequest.sort(s -> s.field(f -> f.field(name).order(sortOrder))); + } } } } - searchSourceBuilder.version(true); - searchRequest.source(searchSourceBuilder); - SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); - + searchRequest.version(true); + SearchResponse response = esClient.search(searchRequest.build(), clazz); if (size == -1) { + List> hits = response.hits().hits(); + String scrollId = response.scrollId(); // Scroll until no more hits are returned - while (true) { - - for (SearchHit searchHit : response.getHits().getHits()) { - // add hit to results - String sourceAsString = searchHit.getSourceAsString(); - final T value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz); - setMetadata(value, searchHit.getId(), searchHit.getVersion(), searchHit.getSeqNo(), searchHit.getPrimaryTerm(), searchHit.getIndex()); + while (!hits.isEmpty()) { + for (Hit hit : hits) { + T value = hit.source(); + setMetadata(value, hit.id(), hit.version(), hit.seqNo(), hit.primaryTerm(), hit.index()); results.add(value); } - SearchScrollRequest searchScrollRequest = new SearchScrollRequest(response.getScrollId()); - searchScrollRequest.scroll(keepAlive); - response = client.scroll(searchScrollRequest, RequestOptions.DEFAULT); + ScrollRequest scrollRequest = new ScrollRequest.Builder().scrollId(scrollId).scroll(keepAlive).build(); - // If we have no more hits, exit - if (response.getHits().getHits().length == 0) { - break; - } + ScrollResponse scrollResponse = esClient.scroll(scrollRequest, clazz); + hits = scrollResponse.hits().hits(); + scrollId = scrollResponse.scrollId(); } - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(response.getScrollId()); - client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + + esClient.clearScroll(new ClearScrollRequest.Builder().scrollId(response.scrollId()).build()); } else { - SearchHits searchHits = response.getHits(); - scrollIdentifier = response.getScrollId(); - totalHits = searchHits.getTotalHits().value; - totalHitsRelation = getTotalHitsRelation(searchHits.getTotalHits()); + totalHits = response.hits().total() != null ? response.hits().total().value() : 0; + totalHitsRelation = getTotalHitsRelation(response.hits().total()); + scrollIdentifier = response.scrollId(); if (scrollIdentifier != null && totalHits == 0) { - // we have no results, we must clear the scroll request immediately. - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(response.getScrollId()); - client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + ClearScrollRequest clearScrollRequest = new ClearScrollRequest.Builder().scrollId(scrollIdentifier).build(); + esClient.clearScroll(clearScrollRequest); } - for (SearchHit searchHit : searchHits) { - String sourceAsString = searchHit.getSourceAsString(); - final T value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz); - setMetadata(value, searchHit.getId(), searchHit.getVersion(), searchHit.getSeqNo(), searchHit.getPrimaryTerm(), searchHit.getIndex()); + + for (Hit hit : response.hits().hits()) { + T value = hit.source(); + setMetadata(value, hit.id(), hit.version() != null ? hit.version() : 0L, hit.seqNo() != null ? hit.seqNo() : 0L, + hit.primaryTerm() != null ? hit.primaryTerm() : 0L, hit.index()); results.add(value); } } @@ -2081,7 +1967,7 @@ protected PartialList execute(Object... args) throws Exception { throw new Exception("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t); } - PartialList result = new PartialList(results, offset, size, totalHits, totalHitsRelation); + PartialList result = new PartialList<>(results, offset, size, totalHits, totalHitsRelation); if (scrollIdentifier != null && totalHits != 0) { result.setScrollIdentifier(scrollIdentifier); result.setScrollTimeValidity(scrollTimeValidity); @@ -2092,86 +1978,100 @@ protected PartialList execute(Object... args) throws Exception { } private PartialList.Relation getTotalHitsRelation(TotalHits totalHits) { - return TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(totalHits.relation) ? PartialList.Relation.GREATER_THAN_OR_EQUAL_TO : PartialList.Relation.EQUAL; + return TotalHitsRelation.Gte.equals(totalHits.relation()) ? + PartialList.Relation.GREATER_THAN_OR_EQUAL_TO : + PartialList.Relation.EQUAL; } - @Override - public PartialList continueScrollQuery(final Class clazz, final String scrollIdentifier, final String scrollTimeValidity) { - return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".continueScrollQuery", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public PartialList continueScrollQuery(final Class clazz, final String scrollIdentifier, + final String scrollTimeValidity) { + return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".continueScrollQuery", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - @Override - protected PartialList execute(Object... args) throws Exception { - List results = new ArrayList(); + @Override protected PartialList execute(Object... args) throws Exception { + List results = new ArrayList<>(); long totalHits = 0; try { - TimeValue keepAlive = TimeValue.parseTimeValue(scrollTimeValidity, TimeValue.timeValueMinutes(10), "scrollTimeValidity"); + Time keepAlive = Time.of(t -> t.time(scrollTimeValidity.isBlank() ? "10m" : scrollTimeValidity)); + + ScrollRequest scrollRequest = new ScrollRequest.Builder().scrollId(scrollIdentifier).scroll(keepAlive).build(); - SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollIdentifier); - searchScrollRequest.scroll(keepAlive); - SearchResponse response = client.scroll(searchScrollRequest, RequestOptions.DEFAULT); + ScrollResponse scrollResponse = esClient.scroll(scrollRequest, clazz); - if (response.getHits().getHits().length == 0) { - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(response.getScrollId()); - client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + if (scrollResponse.hits().hits().isEmpty()) { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest.Builder().scrollId(scrollResponse.scrollId()) + .build(); + esClient.clearScroll(clearScrollRequest); } else { - for (SearchHit searchHit : response.getHits().getHits()) { - // add hit to results - String sourceAsString = searchHit.getSourceAsString(); - final T value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz); - setMetadata(value, searchHit.getId(), searchHit.getVersion(), searchHit.getSeqNo(), searchHit.getPrimaryTerm(), searchHit.getIndex()); + for (Hit hit : scrollResponse.hits().hits()) { + T value = hit.source(); + setMetadata(value, hit.id(), hit.version() != null ? hit.version() : 0L, hit.seqNo() != null ? hit.seqNo() : 0L, + hit.primaryTerm() != null ? hit.primaryTerm() : 0L, hit.index()); results.add(value); } } - PartialList result = new PartialList(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value, getTotalHitsRelation(response.getHits().getTotalHits())); + if (scrollResponse.hits().total() != null) { + totalHits = scrollResponse.hits().total().value(); + } + PartialList result = new PartialList(results, 0, scrollResponse.hits().hits().size(), totalHits, + getTotalHitsRelation(scrollResponse.hits().total())); if (scrollIdentifier != null) { result.setScrollIdentifier(scrollIdentifier); result.setScrollTimeValidity(scrollTimeValidity); } return result; } catch (Exception t) { - throw new Exception("Error continuing scrolling query for itemType=" + clazz.getName() + " scrollIdentifier=" + scrollIdentifier + " scrollTimeValidity=" + scrollTimeValidity, t); + throw new Exception( + "Error continuing scrolling query for itemType=" + clazz.getName() + " scrollIdentifier=" + scrollIdentifier + + " scrollTimeValidity=" + scrollTimeValidity, t); } } }.catchingExecuteInClassLoader(true); } - @Override - public PartialList continueCustomItemScrollQuery(final String customItemType, final String scrollIdentifier, final String scrollTimeValidity) { - return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".continueScrollQuery", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public PartialList continueCustomItemScrollQuery(final String customItemType, final String scrollIdentifier, + final String scrollTimeValidity) { + return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".continueScrollQuery", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - @Override - protected PartialList execute(Object... args) throws Exception { + @Override protected PartialList execute(Object... args) throws Exception { List results = new ArrayList(); long totalHits = 0; try { - TimeValue keepAlive = TimeValue.parseTimeValue(scrollTimeValidity, TimeValue.timeValueMinutes(10), "scrollTimeValidity"); - SearchScrollRequest searchScrollRequest = new SearchScrollRequest(scrollIdentifier); - searchScrollRequest.scroll(keepAlive); - SearchResponse response = client.scroll(searchScrollRequest, RequestOptions.DEFAULT); + Time keepAlive = Time.of(t -> t.time(scrollTimeValidity.isBlank() ? "10m" : scrollTimeValidity)); - if (response.getHits().getHits().length == 0) { - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(response.getScrollId()); - client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + ScrollRequest scrollRequest = new ScrollRequest.Builder().scrollId(scrollIdentifier).scroll(keepAlive).build(); + + ScrollResponse scrollResponse = esClient.scroll(scrollRequest, CustomItem.class); + + if (scrollResponse.hits().hits().isEmpty()) { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest.Builder().scrollId(scrollResponse.scrollId()) + .build(); + esClient.clearScroll(clearScrollRequest); } else { - for (SearchHit searchHit : response.getHits().getHits()) { - // add hit to results - String sourceAsString = searchHit.getSourceAsString(); - final CustomItem value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, CustomItem.class); - setMetadata(value, searchHit.getId(), searchHit.getVersion(), searchHit.getSeqNo(), searchHit.getPrimaryTerm(), searchHit.getIndex()); + for (Hit hit : scrollResponse.hits().hits()) { + CustomItem value = hit.source(); + setMetadata(value, hit.id(), hit.version() != null ? hit.version() : 0L, hit.seqNo() != null ? hit.seqNo() : 0L, + hit.primaryTerm() != null ? hit.primaryTerm() : 0L, hit.index()); results.add(value); } } - PartialList result = new PartialList(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value, getTotalHitsRelation(response.getHits().getTotalHits())); + if (scrollResponse.hits().total() != null) { + totalHits = scrollResponse.hits().total().value(); + } + + PartialList result = new PartialList(results, 0, scrollResponse.hits().hits().size(), totalHits, + getTotalHitsRelation(scrollResponse.hits().total())); if (scrollIdentifier != null) { result.setScrollIdentifier(scrollIdentifier); result.setScrollTimeValidity(scrollTimeValidity); } return result; } catch (Exception t) { - throw new Exception("Error continuing scrolling query for itemType=" + customItemType + " scrollIdentifier=" + scrollIdentifier + " scrollTimeValidity=" + scrollTimeValidity, t); + throw new Exception( + "Error continuing scrolling query for itemType=" + customItemType + " scrollIdentifier=" + scrollIdentifier + + " scrollTimeValidity=" + scrollTimeValidity, t); } } }.catchingExecuteInClassLoader(true); @@ -2180,191 +2080,220 @@ protected PartialList execute(Object... args) throws Exception { /** * @deprecated As of version 1.3.0-incubating, use {@link #aggregateWithOptimizedQuery(Condition, BaseAggregate, String)} instead */ - @Deprecated - @Override - public Map aggregateQuery(Condition filter, BaseAggregate aggregate, String itemType) { + @Deprecated @Override public Map aggregateQuery(Condition filter, BaseAggregate aggregate, String itemType) { return aggregateQuery(filter, aggregate, itemType, false, aggregateQueryBucketSize); } - @Override - public Map aggregateWithOptimizedQuery(Condition filter, BaseAggregate aggregate, String itemType) { + @Override public Map aggregateWithOptimizedQuery(Condition filter, BaseAggregate aggregate, String itemType) { return aggregateQuery(filter, aggregate, itemType, true, aggregateQueryBucketSize); } - @Override - public Map aggregateWithOptimizedQuery(Condition filter, BaseAggregate aggregate, String itemType, int size) { + @Override public Map aggregateWithOptimizedQuery(Condition filter, BaseAggregate aggregate, String itemType, int size) { return aggregateQuery(filter, aggregate, itemType, true, size); } private Map aggregateQuery(final Condition filter, final BaseAggregate aggregate, final String itemType, - final boolean optimizedQuery, int queryBucketSize) { - return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".aggregateQuery", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + final boolean optimizedQuery, int queryBucketSize) { + return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".aggregateQuery", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - @Override - protected Map execute(Object... args) throws IOException { - Map results = new LinkedHashMap(); - - SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); - searchSourceBuilder.size(0); - MatchAllQueryBuilder matchAll = QueryBuilders.matchAllQuery(); - boolean isItemTypeSharingIndex = isItemTypeSharingIndex(itemType); - searchSourceBuilder.query(isItemTypeSharingIndex ? getItemTypeQueryBuilder(itemType) : matchAll); - List lastAggregation = new ArrayList(); + @Override protected Map execute(Object... args) throws IOException { + Map results = new LinkedHashMap<>(); + Map aggregationsByType = new HashMap<>(); if (aggregate != null) { - AggregationBuilder bucketsAggregation = null; + Aggregation bucketsAggregation = null; String fieldName = aggregate.getField(); - if (aggregate instanceof DateAggregate) { - DateAggregate dateAggregate = (DateAggregate) aggregate; - DateHistogramAggregationBuilder dateHistogramBuilder = AggregationBuilders.dateHistogram("buckets").field(fieldName).calendarInterval(new DateHistogramInterval((dateAggregate.getInterval()))); + if (aggregate instanceof DateAggregate dateAggregate) { + DateHistogramAggregation.Builder dateHistogramBuilder = new DateHistogramAggregation.Builder().field(fieldName) + .calendarInterval(CalendarInterval.valueOf(dateAggregate.getIntervalByAlias(dateAggregate.getInterval()))); if (dateAggregate.getFormat() != null) { dateHistogramBuilder.format(dateAggregate.getFormat()); } - bucketsAggregation = dateHistogramBuilder; - } else if (aggregate instanceof NumericRangeAggregate) { - RangeAggregationBuilder rangebuilder = AggregationBuilders.range("buckets").field(fieldName); - for (NumericRange range : ((NumericRangeAggregate) aggregate).getRanges()) { - if (range != null) { - if (range.getFrom() != null && range.getTo() != null) { - rangebuilder.addRange(range.getKey(), range.getFrom(), range.getTo()); - } else if (range.getFrom() != null) { - rangebuilder.addUnboundedFrom(range.getKey(), range.getFrom()); - } else if (range.getTo() != null) { - rangebuilder.addUnboundedTo(range.getKey(), range.getTo()); - } + bucketsAggregation = new Aggregation.Builder().dateHistogram(dateHistogramBuilder.build()).build(); + } else if (aggregate instanceof NumericRangeAggregate numericRangeAggregate) { + List ranges = new ArrayList<>(); + for (NumericRange numericRange : numericRangeAggregate.getRanges()) { + if (numericRange != null) { + ranges.add(AggregationRange.of(builder -> builder.from(numericRange.getFrom()).to(numericRange.getTo()) + .key(numericRange.getKey()))); } } - bucketsAggregation = rangebuilder; - } else if (aggregate instanceof DateRangeAggregate) { - DateRangeAggregate dateRangeAggregate = (DateRangeAggregate) aggregate; - DateRangeAggregationBuilder rangebuilder = AggregationBuilders.dateRange("buckets").field(fieldName); - if (dateRangeAggregate.getFormat() != null) { - rangebuilder.format(dateRangeAggregate.getFormat()); - } + RangeAggregation rangeAgg = new RangeAggregation.Builder().field(fieldName).ranges(ranges).build(); + bucketsAggregation = new Aggregation.Builder().range(rangeAgg).build(); + } else if (aggregate instanceof DateRangeAggregate dateRangeAggregate) { + List dateRanges = new ArrayList<>(); for (DateRange range : dateRangeAggregate.getDateRanges()) { if (range != null) { - rangebuilder.addRange(range.getKey(), range.getFrom() != null ? range.getFrom().toString() : null, range.getTo() != null ? range.getTo().toString() : null); + DateRangeExpression.Builder exprBuilder = new DateRangeExpression.Builder(); + if (range.getKey() != null) { + exprBuilder.key(range.getKey()); + } + if (range.getFrom() != null) { + exprBuilder.from(FieldDateMath.of(f -> f.expr(range.getFrom().toString()))); + } + if (range.getTo() != null) { + exprBuilder.to(FieldDateMath.of(f -> f.expr(range.getTo().toString()))); + } + dateRanges.add(exprBuilder.build()); } } - bucketsAggregation = rangebuilder; - } else if (aggregate instanceof IpRangeAggregate) { - IpRangeAggregate ipRangeAggregate = (IpRangeAggregate) aggregate; - IpRangeAggregationBuilder rangebuilder = AggregationBuilders.ipRange("buckets").field(fieldName); + DateRangeAggregation.Builder dateRangeBuilder = new DateRangeAggregation.Builder().field(fieldName) + .ranges(dateRanges); + + if (dateRangeAggregate.getFormat() != null) { + dateRangeBuilder.format(dateRangeAggregate.getFormat()); + } + + bucketsAggregation = new Aggregation.Builder().dateRange(dateRangeBuilder.build()).build(); + } else if (aggregate instanceof IpRangeAggregate ipRangeAggregate) { + IpRangeAggregation.Builder ipRangeBuilder = new IpRangeAggregation.Builder().field(fieldName); + List ranges = new ArrayList<>(); for (IpRange range : ipRangeAggregate.getRanges()) { if (range != null) { - rangebuilder.addRange(range.getKey(), range.getFrom(), range.getTo()); + IpRangeAggregationRange.of(builder -> builder.from(range.getFrom()).to(range.getTo())); + ranges.add(IpRangeAggregationRange.of(builder -> builder.from(range.getFrom()).to(range.getTo()))); } } - bucketsAggregation = rangebuilder; + ipRangeBuilder.ranges(ranges); + bucketsAggregation = ipRangeBuilder.build()._toAggregation(); } else { fieldName = getPropertyNameWithData(fieldName, itemType); //default if (fieldName != null) { - bucketsAggregation = AggregationBuilders.terms("buckets").field(fieldName).size(queryBucketSize); - if (aggregate instanceof TermsAggregate) { - TermsAggregate termsAggregate = (TermsAggregate) aggregate; + TermsAggregation.Builder termsAggBuilder = new TermsAggregation.Builder().field(fieldName) + .size(queryBucketSize); + if (aggregate instanceof TermsAggregate termsAggregate) { + if (termsAggregate.getPartition() > -1 && termsAggregate.getNumPartitions() > -1) { - ((TermsAggregationBuilder) bucketsAggregation).includeExclude(new IncludeExclude(termsAggregate.getPartition(), termsAggregate.getNumPartitions())); + termsAggBuilder.include(TermsInclude.of(ti -> ti.partition( + pi -> pi.partition(termsAggregate.getPartition()) + .numPartitions(termsAggregate.getNumPartitions())))); } } - } else { - // field name could be null if no existing data exists + bucketsAggregation = termsAggBuilder.build()._toAggregation(); } } if (bucketsAggregation != null) { - final MissingAggregationBuilder missingBucketsAggregation = AggregationBuilders.missing("missing").field(fieldName); - for (AggregationBuilder aggregationBuilder : lastAggregation) { - bucketsAggregation.subAggregation(aggregationBuilder); - missingBucketsAggregation.subAggregation(aggregationBuilder); - } - lastAggregation = Arrays.asList(bucketsAggregation, missingBucketsAggregation); + MissingAggregation missingAggregation = new MissingAggregation.Builder().field(fieldName).build(); + aggregationsByType.put("buckets", bucketsAggregation); + aggregationsByType.put("missing", missingAggregation._toAggregation()); } } + SearchRequest.Builder searchSourceBuilder = new SearchRequest.Builder(); + searchSourceBuilder.index(getIndexNameForQuery(itemType)); + searchSourceBuilder.size(0); + searchSourceBuilder.query( + isItemTypeSharingIndex(itemType) ? getItemTypeQuery(itemType) : Query.of(q -> q.matchAll(m -> m))); + // If the request is optimized then we don't need a global aggregation which is very slow and we can put the query with a // filter on range items in the query block so we don't retrieve all the document before filtering the whole if (optimizedQuery) { - for (AggregationBuilder aggregationBuilder : lastAggregation) { - searchSourceBuilder.aggregation(aggregationBuilder); - } + searchSourceBuilder.aggregations(aggregationsByType); if (filter != null) { searchSourceBuilder.query(wrapWithItemTypeQuery(itemType, conditionESQueryBuilderDispatcher.buildFilter(filter))); } } else { if (filter != null) { - AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", - wrapWithItemTypeQuery(itemType, conditionESQueryBuilderDispatcher.buildFilter(filter))); - for (AggregationBuilder aggregationBuilder : lastAggregation) { - filterAggregation.subAggregation(aggregationBuilder); - } - lastAggregation = Collections.singletonList(filterAggregation); - } + Aggregation.Builder aggBuilder = new Aggregation.Builder(); + aggBuilder.filter(wrapWithItemTypeQuery(itemType, conditionESQueryBuilderDispatcher.buildFilter(filter))) + .aggregations(aggregationsByType); - AggregationBuilder globalAggregation = AggregationBuilders.global("global"); - for (AggregationBuilder aggregationBuilder : lastAggregation) { - globalAggregation.subAggregation(aggregationBuilder); + aggregationsByType = Map.of("filter", aggBuilder.build()); } - - searchSourceBuilder.aggregation(globalAggregation); + Aggregation globalAgg = new Aggregation.Builder().global(new GlobalAggregation.Builder().build()) + .aggregations(aggregationsByType).build(); + searchSourceBuilder.aggregations(String.valueOf(globalAgg._kind()), globalAgg); } - searchRequest.source(searchSourceBuilder); - RequestOptions.Builder builder = RequestOptions.DEFAULT.toBuilder(); + RestClientOptions additionalOptions = null; if (aggQueryMaxResponseSizeHttp != null) { - builder.setHttpAsyncResponseConsumerFactory( - new HttpAsyncResponseConsumerFactory - .HeapBufferedResponseConsumerFactory(aggQueryMaxResponseSizeHttp)); + HttpAsyncResponseConsumerFactory httpAsyncResponseConsumerFactory = new HttpAsyncResponseConsumerFactory.HeapBufferedResponseConsumerFactory( + aggQueryMaxResponseSizeHttp); + RequestOptions requestOptions = RequestOptions.DEFAULT.toBuilder() + .setHttpAsyncResponseConsumerFactory(httpAsyncResponseConsumerFactory).build(); + + additionalOptions = new RestClientOptions(requestOptions, true); } - SearchResponse response = client.search(searchRequest, builder.build()); - Aggregations aggregations = response.getAggregations(); + SearchResponse response; + if (additionalOptions != null) { + ElasticsearchClient clientWithOptions = esClient.withTransportOptions(additionalOptions); + response = clientWithOptions.search(searchSourceBuilder.build()); + clientWithOptions.close(); + } else { + response = esClient.search(searchSourceBuilder.build()); + } + Map aggregations = response.aggregations(); if (aggregations != null) { - if (optimizedQuery) { - if (response.getHits() != null) { - results.put("_filtered", response.getHits().getTotalHits().value); + if (response.hits() != null) { + results.put("_filtered", response.hits().total().value()); } } else { - Global globalAgg = aggregations.get("global"); - results.put("_all", globalAgg.getDocCount()); - aggregations = globalAgg.getAggregations(); - - if (aggregations.get("filter") != null) { - Filter filterAgg = aggregations.get("filter"); - results.put("_filtered", filterAgg.getDocCount()); - aggregations = filterAgg.getAggregations(); + GlobalAggregate globalAggregate = aggregations.get(Aggregation.Kind.Global.jsonValue()).global(); + results.put("_all", globalAggregate.docCount()); + aggregations = globalAggregate.aggregations(); + + if (aggregations.get(Aggregate.Kind.Filter.jsonValue()) != null) { + FilterAggregate filterAggregate = aggregations.get(Aggregation.Kind.Filter.jsonValue()).filter(); + results.put("_filtered", filterAggregate.docCount()); + aggregations = filterAggregate.aggregations(); } } if (aggregations.get("buckets") != null) { - if (aggQueryThrowOnMissingDocs) { - if (aggregations.get("buckets") instanceof Terms) { - Terms terms = aggregations.get("buckets"); - if (terms.getDocCountError() > 0 || terms.getSumOfOtherDocCounts() > 0) { - throw new UnsupportedOperationException("Some docs are missing in aggregation query. docCountError is:" + - terms.getDocCountError() + " sumOfOtherDocCounts:" + terms.getSumOfOtherDocCounts()); + Aggregate agg = aggregations.get("buckets"); + if (agg.isSterms()) { + StringTermsAggregate terms = aggregations.get("buckets").sterms(); + if (terms.docCountErrorUpperBound() > 0 || terms.sumOtherDocCount() > 0) { + throw new UnsupportedOperationException("Some docs are missing in aggregation query. docCountError is:" + + terms.docCountErrorUpperBound() + " sumOfOtherDocCounts:" + terms.sumOtherDocCount()); } } + // TODO check if needed for dTerms and lTerms } long totalDocCount = 0; - MultiBucketsAggregation terms = aggregations.get("buckets"); - for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) { - results.put(bucket.getKeyAsString(), bucket.getDocCount()); - totalDocCount += bucket.getDocCount(); + Aggregate bucketsAggregate = aggregations.get("buckets"); + if (bucketsAggregate.isMultiTerms()) { + MultiTermsAggregate terms = aggregations.get("buckets").multiTerms(); + for (MultiTermsBucket bucket : terms.buckets().array()) { + results.put(bucket.keyAsString(), bucket.docCount()); + totalDocCount += bucket.docCount(); + } + } else if (bucketsAggregate.isSterms()) { + StringTermsAggregate terms = bucketsAggregate.sterms(); + for (StringTermsBucket bucket : terms.buckets().array()) { + results.put(bucket.key().stringValue(), bucket.docCount()); + totalDocCount += bucket.docCount(); + } + } else if (bucketsAggregate.isLterms()) { + LongTermsAggregate terms = bucketsAggregate.lterms(); + for (LongTermsBucket bucket : terms.buckets().array()) { + results.put(bucket.keyAsString(), bucket.docCount()); + totalDocCount += bucket.docCount(); + } + } else if (bucketsAggregate.isDateHistogram()){ + DateHistogramAggregate histogramAggregate = bucketsAggregate.dateHistogram(); + for (DateHistogramBucket bucket : histogramAggregate.buckets().array()) { + results.put(bucket.keyAsString(), bucket.docCount()); + totalDocCount += bucket.docCount(); + } } - SingleBucketAggregation missing = aggregations.get("missing"); - if (missing.getDocCount() > 0) { - results.put("_missing", missing.getDocCount()); - totalDocCount += missing.getDocCount(); + + MissingAggregate missing = aggregations.get("missing").missing(); + if (missing.docCount() > 0) { + results.put("_missing", missing.docCount()); + totalDocCount += missing.docCount(); } - if (response.getHits() != null && TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(response.getHits().getTotalHits().relation)) { + if (response.hits() != null && TotalHitsRelation.Gte.equals(response.hits().total().relation())) { results.put("_filtered", totalDocCount); } } @@ -2383,70 +2312,72 @@ private String[] getRouting(String fieldName, String[] fieldVal return routing; } - @Override - public void refresh() { - new InClassLoaderExecute(metricsService, this.getClass().getName() + ".refresh", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public void refresh() { + new InClassLoaderExecute(metricsService, this.getClass().getName() + ".refresh", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) { - if (bulkProcessor != null) { - bulkProcessor.flush(); - } try { - client.indices().refresh(Requests.refreshRequest(), RequestOptions.DEFAULT); + esClient.indices().refresh(new RefreshRequest.Builder().build()); } catch (IOException e) { - e.printStackTrace(); //TODO manage ES7 + LOGGER.error("Failed to refresh persistence for reason: {}. Set the log in DEBUG level for details", e.getMessage()); + LOGGER.debug("Error on refresh: ", e); } return true; } }.catchingExecuteInClassLoader(true); } - @Override - public void refreshIndex(Class clazz, Date dateHint) { - new InClassLoaderExecute(metricsService, this.getClass().getName() + ".refreshIndex", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public void refreshIndex(Class clazz, Date dateHint) { + new InClassLoaderExecute(metricsService, this.getClass().getName() + ".refreshIndex", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) { try { - String itemType = Item.getItemType(clazz); - String index = getIndex(itemType); - client.indices().refresh(Requests.refreshRequest(index), RequestOptions.DEFAULT); + esClient.indices().refresh(RefreshRequest.of(builder -> builder.index(getIndex(Item.getItemType(clazz))))); } catch (IOException e) { - e.printStackTrace(); //TODO manage ES7 + LOGGER.error("Failed to refresh index for reason: {}. Set the log in DEBUG level for details", e.getMessage()); + LOGGER.debug("Error on refresh: ", e); } return true; } }.catchingExecuteInClassLoader(true); } - - @Override - public void purge(final Date date) { + @Override public void purge(final Date date) { // nothing, this method is deprecated since 2.2.0 } - @Override - public void purgeTimeBasedItems(int existsNumberOfDays, Class clazz) { - new InClassLoaderExecute(metricsService, this.getClass().getName() + ".purgeTimeBasedItems", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public void purgeTimeBasedItems(int existsNumberOfDays, Class clazz) { + new InClassLoaderExecute(metricsService, this.getClass().getName() + ".purgeTimeBasedItems", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { protected Boolean execute(Object... args) throws Exception { String itemType = Item.getItemType(clazz); if (existsNumberOfDays > 0 && isItemTypeRollingOver(itemType)) { // First we purge the documents - removeByQuery(QueryBuilders.rangeQuery("timeStamp").lte("now-" + existsNumberOfDays + "d"), clazz); + Query query = Query.of(builder -> builder.range( + RangeQuery.of(r -> r.term(term -> term.field("timeStamp").lte("now-" + existsNumberOfDays + "d"))))); + removeByQuery(query, clazz); // get count per index for those time based data TreeMap countsPerIndex = new TreeMap<>(); - GetIndexResponse getIndexResponse = client.indices().get(new GetIndexRequest(getIndexNameForQuery(itemType)), RequestOptions.DEFAULT); - for (String index : getIndexResponse.getIndices()) { - countsPerIndex.put(index, client.count(new CountRequest(index), RequestOptions.DEFAULT).getCount()); + GetIndexResponse getIndexResponse = esClient.indices() + .get(new GetIndexRequest.Builder().index(getIndexNameForQuery(itemType)).build()); + Map indices = getIndexResponse.indices(); + + for (Map.Entry entry : indices.entrySet()) { + String indexName = entry.getKey(); + CountRequest countRequest = new CountRequest.Builder().index(indexName).build(); + countsPerIndex.put(indexName, esClient.count(countRequest).count()); } // Check for count=0 and remove them - if (countsPerIndex.size() >= 1) { + if (!countsPerIndex.isEmpty()) { // do not check the last index, because it's the one used to write documents countsPerIndex.pollLastEntry(); for (Map.Entry indexCount : countsPerIndex.entrySet()) { if (indexCount.getValue() == 0) { - client.indices().delete(new DeleteIndexRequest(indexCount.getKey()), RequestOptions.DEFAULT); + esClient.indices().delete(new DeleteIndexRequest.Builder().index(indexCount.getKey()).build()); } } } @@ -2457,52 +2388,50 @@ protected Boolean execute(Object... args) throws Exception { }.catchingExecuteInClassLoader(true); } - @Override - public void purge(final String scope) { + @Override public void purge(final String scope) { LOGGER.debug("Purge scope {}", scope); - new InClassLoaderExecute(metricsService, this.getClass().getName() + ".purgeWithScope", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - @Override - protected Void execute(Object... args) throws IOException { - QueryBuilder query = termQuery("scope", scope); + new InClassLoaderExecute(metricsService, this.getClass().getName() + ".purgeWithScope", this.bundleContext, + this.fatalIllegalStateErrors, throwExceptions) { + @Override protected Void execute(Object... args) throws IOException { + Query query = TermQuery.of(builder -> builder.field("scope").value(scope))._toQuery(); - BulkRequest deleteByScopeBulkRequest = new BulkRequest(); + List operations = new ArrayList<>(); - final TimeValue keepAlive = TimeValue.timeValueHours(1); - SearchRequest searchRequest = new SearchRequest(getAllIndexForQuery()).scroll(keepAlive); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - .query(query) - .size(100); - searchRequest.source(searchSourceBuilder); - SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); + Time keepAlive = Time.of(t -> t.time("1h")); - // Scroll until no more hits are returned - while (true) { + SearchRequest searchRequest = SearchRequest.of( + s -> s.index(getAllIndexForQuery()).scroll(keepAlive).size(100).query(query).source(src -> src.fetch(true))); + SearchResponse searchResponse = esClient.search(searchRequest, JsonData.class); - for (SearchHit hit : response.getHits().getHits()) { + List> hits = searchResponse.hits().hits(); + String scrollId = searchResponse.scrollId(); + // Scroll until no more hits are returned + while (!hits.isEmpty()) { + for (Hit hit : searchResponse.hits().hits()) { // add hit to bulk delete - DeleteRequest deleteRequest = new DeleteRequest(hit.getIndex(), hit.getId()); - deleteByScopeBulkRequest.add(deleteRequest); + operations.add(BulkOperation.of(builder -> builder.delete(d -> d.index(hit.index()).id(hit.id())))); } - SearchScrollRequest searchScrollRequest = new SearchScrollRequest(response.getScrollId()); - searchScrollRequest.scroll(keepAlive); - response = client.scroll(searchScrollRequest, RequestOptions.DEFAULT); - + ScrollRequest scrollRequest = new ScrollRequest.Builder().scrollId(scrollId).scroll(keepAlive).build(); + ScrollResponse scrollResponse = esClient.scroll(scrollRequest, JsonData.class); + hits = scrollResponse.hits().hits(); + scrollId = scrollResponse.scrollId(); // If we have no more hits, exit - if (response.getHits().getHits().length == 0) { - ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); - clearScrollRequest.addScrollId(response.getScrollId()); - client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); - break; + if (hits.isEmpty()) { + ClearScrollRequest clearScrollRequest = new ClearScrollRequest.Builder().scrollId(scrollId).build(); + esClient.clearScroll(clearScrollRequest); } } // we're done with the scrolling, delete now - if (deleteByScopeBulkRequest.numberOfActions() > 0) { - final BulkResponse deleteResponse = client.bulk(deleteByScopeBulkRequest, RequestOptions.DEFAULT); - if (deleteResponse.hasFailures()) { - // do something - LOGGER.warn("Couldn't delete from scope {}:\n{}", scope, deleteResponse.buildFailureMessage()); + if (!operations.isEmpty()) { + BulkResponse bulkResponse = esClient.bulk(b -> b.operations(operations)); + if (bulkResponse.errors()) { + bulkResponse.items().forEach(item -> { + if (item.error() != null) { + LOGGER.warn("Couldn't delete item {} from scope {}: {}", item.id(), scope, item.error().reason()); + } + }); } } return null; @@ -2510,57 +2439,62 @@ protected Void execute(Object... args) throws IOException { }.catchingExecuteInClassLoader(true); } - @Override - public Map getSingleValuesMetrics(final Condition condition, final String[] metrics, final String field, final String itemType) { - return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".getSingleValuesMetrics", this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { + @Override public Map getSingleValuesMetrics(final Condition condition, final String[] metrics, final String field, + final String itemType) { + return new InClassLoaderExecute>(metricsService, this.getClass().getName() + ".getSingleValuesMetrics", + this.bundleContext, this.fatalIllegalStateErrors, throwExceptions) { - @Override - protected Map execute(Object... args) throws IOException { + @Override protected Map execute(Object... args) throws IOException { Map results = new LinkedHashMap(); - SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType)); - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - .size(0) - .query(isItemTypeSharingIndex(itemType) ? getItemTypeQueryBuilder(itemType) : QueryBuilders.matchAllQuery()); - - AggregationBuilder filterAggregation = AggregationBuilders.filter("metrics", conditionESQueryBuilderDispatcher.buildFilter(condition)); - + Map subAggs = new HashMap<>(); if (metrics != null) { for (String metric : metrics) { switch (metric) { case "sum": - filterAggregation.subAggregation(AggregationBuilders.sum("sum").field(field)); + subAggs.put("sum", AggregationBuilders.sum().field(field).build()._toAggregation()); break; case "avg": - filterAggregation.subAggregation(AggregationBuilders.avg("avg").field(field)); + subAggs.put("avg", AggregationBuilders.avg().field(field).build()._toAggregation()); break; case "min": - filterAggregation.subAggregation(AggregationBuilders.min("min").field(field)); + subAggs.put("min", AggregationBuilders.min().field(field).build()._toAggregation()); break; case "max": - filterAggregation.subAggregation(AggregationBuilders.max("max").field(field)); + subAggs.put("max", AggregationBuilders.max().field(field).build()._toAggregation()); break; case "card": - filterAggregation.subAggregation(AggregationBuilders.cardinality("card").field(field)); + subAggs.put("card", AggregationBuilders.cardinality().field(field).build()._toAggregation()); break; case "count": - filterAggregation.subAggregation(AggregationBuilders.count("count").field(field)); + subAggs.put("count", Aggregation.of(a -> a.valueCount(vc -> vc.field(field)))); break; } } } - searchSourceBuilder.aggregation(filterAggregation); - searchRequest.source(searchSourceBuilder); - SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT); - Aggregations aggregations = response.getAggregations(); - if (aggregations != null) { - Aggregation metricsResults = aggregations.get("metrics"); - if (metricsResults instanceof HasAggregations) { - aggregations = ((HasAggregations) metricsResults).getAggregations(); - for (Aggregation aggregation : aggregations) { - NumericMetricsAggregation.SingleValue singleValue = (NumericMetricsAggregation.SingleValue) aggregation; - results.put("_" + singleValue.getName(), singleValue.value()); + Aggregation filterAggregation = Aggregation.of( + a -> a.filter(conditionESQueryBuilderDispatcher.buildFilter(condition)).aggregations(subAggs)); + SearchRequest searchRequest = SearchRequest.of( + s -> s.index(getIndexNameForQuery(itemType)).size(0).source(builder -> builder.fetch(true)) + .query(isItemTypeSharingIndex(itemType) ? getItemTypeQuery(itemType) : Query.of(q -> q.matchAll(m -> m))) + .aggregations("metrics", filterAggregation)); + + SearchResponse searchResponse = esClient.search(searchRequest); + + Map aggregationResult = searchResponse.aggregations(); + + if (aggregationResult != null) { + Aggregate metricsAgg = aggregationResult.get("metrics"); + if (metricsAgg != null && metricsAgg.isFilter()) { + Map subAggsResult = metricsAgg.filter().aggregations(); + for (Map.Entry entry : subAggsResult.entrySet()) { + String name = entry.getKey(); + Double value = getAggregeValue(entry); + + if (value != null) { + results.put("_" + name, value); + } } } } @@ -2569,9 +2503,27 @@ protected Map execute(Object... args) throws IOException { }.catchingExecuteInClassLoader(true); } + private static Double getAggregeValue(Map.Entry entry) { + Aggregate agg = entry.getValue(); + + Double value = null; + if (agg.isSum()) { + value = agg.sum().value(); + } else if (agg.isAvg()) { + value = agg.avg().value(); + } else if (agg.isMin()) { + value = agg.min().value(); + } else if (agg.isMax()) { + value = agg.max().value(); + } else if (agg.isCardinality()) { + value = (double) agg.cardinality().value(); + } else if (agg.isValueCount()) { + value = agg.valueCount().value(); + } + return value; + } - private String getConfig(Map settings, String key, - String defaultValue) { + private String getConfig(Map settings, String key, String defaultValue) { if (settings != null && settings.get(key) != null) { return settings.get(key); } @@ -2580,13 +2532,14 @@ private String getConfig(Map settings, String key, public abstract static class InClassLoaderExecute { - private String timerName; - private MetricsService metricsService; - private BundleContext bundleContext; - private String[] fatalIllegalStateErrors; // Errors that if occur - stop the application - private boolean throwExceptions; + private final String timerName; + private final MetricsService metricsService; + private final BundleContext bundleContext; + private final String[] fatalIllegalStateErrors; // Errors that if occur - stop the application + private final boolean throwExceptions; - public InClassLoaderExecute(MetricsService metricsService, String timerName, BundleContext bundleContext, String[] fatalIllegalStateErrors, boolean throwExceptions) { + public InClassLoaderExecute(MetricsService metricsService, String timerName, BundleContext bundleContext, + String[] fatalIllegalStateErrors, boolean throwExceptions) { this.timerName = timerName; this.metricsService = metricsService; this.bundleContext = bundleContext; @@ -2618,7 +2571,8 @@ public T catchingExecuteInClassLoader(boolean logError, Object... args) { Throwable tTemp = t; // Go over the stack trace and check if there were any fatal state errors while (tTemp != null) { - if (tTemp instanceof IllegalStateException && Arrays.stream(this.fatalIllegalStateErrors).anyMatch(tTemp.getMessage()::contains)) { + if (tTemp instanceof IllegalStateException && Arrays.stream(this.fatalIllegalStateErrors) + .anyMatch(tTemp.getMessage()::contains)) { handleFatalStateError(); // Stop application return null; } @@ -2672,38 +2626,36 @@ private String getDocumentIDForItemType(String itemId, String itemType) { return systemItems.contains(itemType) ? (itemId + "_" + itemType.toLowerCase()) : itemId; } - private QueryBuilder wrapWithItemTypeQuery(String itemType, QueryBuilder originalQuery) { + private Query wrapWithItemTypeQuery(String itemType, Query originalQuery) { if (isItemTypeSharingIndex(itemType)) { - BoolQueryBuilder wrappedQuery = QueryBuilders.boolQuery(); - wrappedQuery.must(getItemTypeQueryBuilder(itemType)); - wrappedQuery.must(originalQuery); - return wrappedQuery; + return Query.of(q -> q.bool(b -> b.must(getItemTypeQuery(itemType)).must(originalQuery))); } return originalQuery; } - private QueryBuilder wrapWithItemsTypeQuery(String[] itemTypes, QueryBuilder originalQuery) { + private Query wrapWithItemsTypeQuery(String[] itemTypes, Query originalQuery) { if (itemTypes.length == 1) { return wrapWithItemTypeQuery(itemTypes[0], originalQuery); } if (Arrays.stream(itemTypes).anyMatch(this::isItemTypeSharingIndex)) { - BoolQueryBuilder itemTypeQuery = QueryBuilders.boolQuery(); - itemTypeQuery.minimumShouldMatch(1); + BoolQuery.Builder itemTypeQuery = new BoolQuery.Builder(); + itemTypeQuery.minimumShouldMatch("1"); + for (String itemType : itemTypes) { - itemTypeQuery.should(getItemTypeQueryBuilder(itemType)); + itemTypeQuery.should(getItemTypeQuery(itemType)); } - BoolQueryBuilder wrappedQuery = QueryBuilders.boolQuery(); - wrappedQuery.filter(itemTypeQuery); + BoolQuery.Builder wrappedQuery = new BoolQuery.Builder(); + wrappedQuery.filter(itemTypeQuery.build()); wrappedQuery.must(originalQuery); - return wrappedQuery; + return Query.of(builder -> builder.bool(wrappedQuery.build())); } return originalQuery; } - private QueryBuilder getItemTypeQueryBuilder(String itemType) { - return QueryBuilders.termQuery("itemType", ConditionContextHelper.foldToASCII(itemType)); + private Query getItemTypeQuery(String itemType) { + return Query.of(q -> q.term(t -> t.field("itemType").value(ConditionContextHelper.foldToASCII(itemType)))); } private boolean isItemTypeSharingIndex(String itemType) { @@ -2711,17 +2663,18 @@ private boolean isItemTypeSharingIndex(String itemType) { } private boolean isItemTypeRollingOver(String itemType) { - return (rolloverIndices != null ? rolloverIndices : itemsMonthlyIndexed).contains(itemType); + return rolloverIndices.contains(itemType); } - private WriteRequest.RefreshPolicy getRefreshPolicy(String itemType) { + private Refresh getRefreshPolicy(String itemType) { if (itemTypeToRefreshPolicy.containsKey(itemType)) { return itemTypeToRefreshPolicy.get(itemType); } - return WriteRequest.RefreshPolicy.NONE; + + return Refresh.False; } - private void logMetadataItemOperation (String operation, Item item) { + private void logMetadataItemOperation(String operation, Item item) { if (item instanceof MetadataItem) { LOGGER.info("Item of type {} with ID {} has been {}", item.getItemType(), item.getItemId(), operation); } diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticsearchClientFactory.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticsearchClientFactory.java new file mode 100644 index 0000000000..5606aad149 --- /dev/null +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticsearchClientFactory.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.elasticsearch; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +import javax.net.ssl.SSLContext; +import java.util.List; + +public class ElasticsearchClientFactory { + + public static ElasticsearchClient createClient( + List hosts, + Integer socketTimeout, + SSLContext sslContext, + String username, + String password) { + + RestClientBuilder builder = RestClient.builder(hosts.toArray(new HttpHost[0])); + + if (socketTimeout != null) { + builder.setRequestConfigCallback(requestConfigBuilder -> + requestConfigBuilder.setSocketTimeout(socketTimeout)); + } + + if (sslContext != null) { + builder.setHttpClientConfigCallback(httpClientBuilder -> + httpClientBuilder.setSSLContext(sslContext)); + } + + if (username != null && password != null) { + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, + new UsernamePasswordCredentials(username, password)); + + builder.setHttpClientConfigCallback(httpClientBuilder -> { + if (sslContext != null) { + httpClientBuilder.setSSLContext(sslContext); + } + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + }); + } + + RestClient restClient = builder.build(); + + ElasticsearchTransport transport = new RestClientTransport( + restClient, new JacksonJsonpMapper(ESCustomObjectMapper.getObjectMapper())); + + return new ElasticsearchClient(transport); + } + + public static ClientBuilder builder() { + return new ClientBuilder(); + } + + public static class ClientBuilder { + private List hosts; + private Integer socketTimeout; + private SSLContext sslContext; + private String username; + private String password; + + public ClientBuilder hosts(List hosts) { + this.hosts = hosts; + return this; + } + + public ClientBuilder socketTimeout(Integer socketTimeout) { + this.socketTimeout = socketTimeout; + return this; + } + + public ClientBuilder sslContext(SSLContext sslContext) { + this.sslContext = sslContext; + return this; + } + + public ClientBuilder usernameAndPassword(String username, String password) { + this.username = username; + this.password = password; + return this; + } + + public ElasticsearchClient build() { + return createClient(hosts, socketTimeout, sslContext, username, password); + } + } +} \ No newline at end of file diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/BooleanConditionESQueryBuilder.java similarity index 64% rename from plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java rename to persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/BooleanConditionESQueryBuilder.java index fa33b29b62..36ae5b6509 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/BooleanConditionESQueryBuilder.java @@ -15,14 +15,13 @@ * limitations under the License. */ -package org.apache.unomi.plugins.baseplugin.conditions; +package org.apache.unomi.persistence.elasticsearch.querybuilders.core; +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,15 +29,15 @@ import java.util.Map; /** - * ES query builder for boolean conditions. + * Elasticsearch query builder for boolean conditions. */ public class BooleanConditionESQueryBuilder implements ConditionESQueryBuilder { private static final Logger LOGGER = LoggerFactory.getLogger(BooleanConditionESQueryBuilder.class.getName()); @Override - public QueryBuilder buildQuery(Condition condition, Map context, - ConditionESQueryBuilderDispatcher dispatcher) { + public Query buildQuery(Condition condition, Map context, + ConditionESQueryBuilderDispatcher dispatcher) { boolean isAndOperator = "and".equalsIgnoreCase((String) condition.getParameter("operator")); @SuppressWarnings("unchecked") List conditions = (List) condition.getParameter("subConditions"); @@ -49,31 +48,35 @@ public QueryBuilder buildQuery(Condition condition, Map context, return dispatcher.buildFilter(conditions.get(0), context); } - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); for (int i = 0; i < conditionCount; i++) { if (isAndOperator) { - QueryBuilder andFilter = dispatcher.buildFilter(conditions.get(i), context); + Query andFilter = dispatcher.buildFilter(conditions.get(i), context); if (andFilter != null) { - if (andFilter.getName().equals("range")) { + if (andFilter.isRange()) { boolQueryBuilder.filter(andFilter); } else { boolQueryBuilder.must(andFilter); } } else { LOGGER.warn("Null filter for boolean AND sub condition. See debug log level for more information"); - if (LOGGER.isDebugEnabled()) LOGGER.debug("Null filter for boolean AND sub condition {}", conditions.get(i)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Null filter for boolean AND sub condition {}", conditions.get(i)); + } } } else { - QueryBuilder orFilter = dispatcher.buildFilter(conditions.get(i), context); + Query orFilter = dispatcher.buildFilter(conditions.get(i), context); if (orFilter != null) { boolQueryBuilder.should(orFilter); } else { LOGGER.warn("Null filter for boolean OR sub condition. See debug log level for more information"); - if (LOGGER.isDebugEnabled()) LOGGER.debug("Null filter for boolean OR sub condition {}", conditions.get(i)); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("Null filter for boolean OR sub condition {}", conditions.get(i)); + } } } } - return boolQueryBuilder; + return Query.of(q->q.bool(boolQueryBuilder.build())); } } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/MatchAllConditionESQueryBuilder.java similarity index 65% rename from plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java rename to persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/MatchAllConditionESQueryBuilder.java index 700286ac46..9ec809cd93 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionESQueryBuilder.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/MatchAllConditionESQueryBuilder.java @@ -15,20 +15,19 @@ * limitations under the License. */ -package org.apache.unomi.plugins.baseplugin.conditions; +package org.apache.unomi.persistence.elasticsearch.querybuilders.core; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import java.util.Map; public class MatchAllConditionESQueryBuilder implements ConditionESQueryBuilder { @Override - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { - return QueryBuilders.matchAllQuery(); + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + return Query.of(q->q.matchAll(m->m)); } } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NestedConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/NestedConditionESQueryBuilder.java similarity index 66% rename from plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NestedConditionESQueryBuilder.java rename to persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/NestedConditionESQueryBuilder.java index e8c3701eb9..df80d13c14 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NestedConditionESQueryBuilder.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/NestedConditionESQueryBuilder.java @@ -15,20 +15,19 @@ * limitations under the License. */ -package org.apache.unomi.plugins.baseplugin.conditions; +package org.apache.unomi.persistence.elasticsearch.querybuilders.core; -import org.apache.lucene.search.join.ScoreMode; +import co.elastic.clients.elasticsearch._types.query_dsl.ChildScoreMode; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import java.util.Map; public class NestedConditionESQueryBuilder implements ConditionESQueryBuilder { @Override - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { String path = (String) condition.getParameter("path"); Condition subCondition = (Condition) condition.getParameter("subCondition"); @@ -36,9 +35,9 @@ public QueryBuilder buildQuery(Condition condition, Map context, throw new IllegalArgumentException("Impossible to build Nested query, subCondition and path properties should be provided"); } - QueryBuilder nestedQueryBuilder = dispatcher.buildFilter(subCondition, context); - if (nestedQueryBuilder != null) { - return QueryBuilders.nestedQuery(path, nestedQueryBuilder, ScoreMode.Avg); + Query nestedQuery = dispatcher.buildFilter(subCondition, context); + if (nestedQuery != null) { + return Query.of(q -> q.nested(n -> n.path(path).query(nestedQuery).scoreMode(ChildScoreMode.Avg))); } else { throw new IllegalArgumentException("Impossible to build Nested query due to subCondition filter null"); } diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/NotConditionESQueryBuilder.java similarity index 65% rename from plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java rename to persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/NotConditionESQueryBuilder.java index 9574fe82f5..724703f19d 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionESQueryBuilder.java +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/NotConditionESQueryBuilder.java @@ -15,13 +15,12 @@ * limitations under the License. */ -package org.apache.unomi.plugins.baseplugin.conditions; +package org.apache.unomi.persistence.elasticsearch.querybuilders.core; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import java.util.Map; @@ -30,8 +29,8 @@ */ public class NotConditionESQueryBuilder implements ConditionESQueryBuilder { - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { Condition subCondition = (Condition) condition.getParameter("subCondition"); - return QueryBuilders.boolQuery().mustNot(dispatcher.buildFilter(subCondition, context)); + return Query.of(q->q.bool(b->b.mustNot(dispatcher.buildFilter(subCondition, context)))); } } diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/PropertyConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/PropertyConditionESQueryBuilder.java new file mode 100644 index 0000000000..df95cb0a42 --- /dev/null +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/PropertyConditionESQueryBuilder.java @@ -0,0 +1,439 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.unomi.persistence.elasticsearch.querybuilders.core; + +import co.elastic.clients.elasticsearch._types.FieldValue; +import co.elastic.clients.elasticsearch._types.GeoDistanceType; +import co.elastic.clients.elasticsearch._types.query_dsl.*; +import co.elastic.clients.util.ObjectBuilder; +import org.apache.commons.lang3.ObjectUtils; +import org.apache.unomi.api.conditions.Condition; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; +import org.apache.unomi.persistence.spi.conditions.ConditionContextHelper; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.ISODateTimeFormat; + +import java.time.OffsetDateTime; +import java.util.*; + +import static org.apache.unomi.persistence.spi.conditions.DateUtils.getDate; + +/** + * Builder to create Elasticsearch queries from property conditions. + * This class handles different types of comparison operators to create the corresponding queries. + */ +public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder { + + private final DateTimeFormatter dateTimeFormatter; + + // Operator groups for better organization + private static final Set EQUALITY_OPERATORS = Set.of("equals", "notEquals"); + private static final Set COMPARISON_OPERATORS = Set.of("greaterThan", "lessThanOrEqualTo", "lessThan", "greaterThanOrEqualTo"); + private static final Set EXISTENCE_OPERATORS = Set.of("exists", "missing"); + private static final Set CONTENT_OPERATORS = Set.of("contains", "notContains", "startsWith", "endsWith", "matchesRegex"); + private static final Set COLLECTION_OPERATORS = Set.of("in", "notIn", "all", "inContains", "hasSomeOf", "hasNoneOf"); + private static final Set DATE_OPERATORS = Set.of("isDay", "isNotDay"); + + public PropertyConditionESQueryBuilder() { + this.dateTimeFormatter = ISODateTimeFormat.dateTime(); + } + + @Override + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + String comparisonOperator = (String) condition.getParameter("comparisonOperator"); + String propertyName = (String) condition.getParameter("propertyName"); + + validateRequiredParameters(comparisonOperator, propertyName); + + // Extract and normalize condition values + PropertyValues propertyValues = extractPropertyValues(condition); + + if (EQUALITY_OPERATORS.contains(comparisonOperator)) { + return buildEqualityQuery(propertyName, propertyValues.singleValue, comparisonOperator); + } else if (COMPARISON_OPERATORS.contains(comparisonOperator)) { + return buildComparisonQuery(propertyName, propertyValues.singleValue, comparisonOperator); + } else if (comparisonOperator.equals("between")) { + return buildBetweenQuery(propertyName, propertyValues.multipleValues); + } else if (EXISTENCE_OPERATORS.contains(comparisonOperator)) { + return buildExistenceQuery(propertyName, comparisonOperator); + } else if (CONTENT_OPERATORS.contains(comparisonOperator)) { + return buildContentQuery(propertyName, (String)propertyValues.singleValue, comparisonOperator); + } else if (COLLECTION_OPERATORS.contains(comparisonOperator)) { + return buildCollectionQuery(propertyName, propertyValues.multipleValues, comparisonOperator); + } else if (DATE_OPERATORS.contains(comparisonOperator)) { + return buildDateQuery(propertyName, propertyValues.singleValue, comparisonOperator); + } else if (comparisonOperator.equals("distance")) { + return buildDistanceQuery(condition, propertyName); + } + + return null; + } + + /** + * Class to group different types of property values + */ + private static class PropertyValues { + final Object singleValue; + final Collection multipleValues; + + PropertyValues(Object singleValue, Collection multipleValues) { + this.singleValue = singleValue; + this.multipleValues = multipleValues; + } + } + + /** + * Extracts and normalizes property values from the condition + */ + private PropertyValues extractPropertyValues(Condition condition) { + String stringValue = ConditionContextHelper.forceFoldToASCII(condition.getParameter("propertyValue")); + Object integerValue = condition.getParameter("propertyValueInteger"); + Object doubleValue = condition.getParameter("propertyValueDouble"); + Object dateValue = convertDateToISO(condition.getParameter("propertyValueDate")); + Object dateExprValue = condition.getParameter("propertyValueDateExpr"); + + Collection stringValues = ConditionContextHelper.forceFoldToASCII((Collection) condition.getParameter("propertyValues")); + Collection integerValues = (Collection) condition.getParameter("propertyValuesInteger"); + Collection doubleValues = (Collection) condition.getParameter("propertyValuesDouble"); + Collection dateValues = convertDatesToISO((Collection) condition.getParameter("propertyValuesDate")); + Collection dateExprValues = (Collection) condition.getParameter("propertyValuesDateExpr"); + + Object singleValue = ObjectUtils.firstNonNull(stringValue, integerValue, doubleValue, dateValue, dateExprValue); + Collection multipleValues = ObjectUtils.firstNonNull(stringValues, integerValues, doubleValues, dateValues, dateExprValues); + + return new PropertyValues(singleValue, multipleValues); + } + + /** + * Validates that required parameters are present + */ + private void validateRequiredParameters(String comparisonOperator, String propertyName) { + if (comparisonOperator == null || propertyName == null) { + throw new IllegalArgumentException( + "Cannot build ES query, the condition is not valid: comparisonOperator and propertyName properties are required"); + } + } + + /** + * Checks that the required value is present + */ + private void checkRequiredValue(Object value, String name, String operator, boolean multiple) { + if (value == null) { + throw new IllegalArgumentException( + "Cannot build ES query, missing value" + (multiple ? "s" : "") + + " for condition using operator: " + operator + + " and property: " + name); + } + } + + /** + * Checks that the collection contains exactly the expected number of elements + */ + private void checkRequiredValuesSize(Collection values, String name, String operator, int expectedSize) { + if (values == null || values.size() != expectedSize) { + throw new IllegalArgumentException( + "Cannot build ES query, missing " + expectedSize + + " values for a condition using operator: " + operator + + " and property: " + name); + } + } + + /** + * Builds an equality query (equals, notEquals) + */ + private Query buildEqualityQuery(String propertyName, Object value, String operator) { + checkRequiredValue(value, propertyName, operator, false); + if (operator.equals("equals")) { + return Query.of(q -> q.term(t -> t.field(propertyName).value(v -> getValue(value)))); + } else { // notEquals + return Query.of(q -> q.bool(b -> b.mustNot(m -> m.term(t -> t.field(propertyName).value(v -> getValue(value)))))); + } + } + + /** + * Builds a comparison query (greaterThan, lessThan, etc.) + */ + private Query buildComparisonQuery(String propertyName, Object value, String operator) { + checkRequiredValue(value, propertyName, operator, false); + return Query.of(q -> q.range(getRangeQuery(propertyName, value, operator))); + } + + /** + * Builds a between query + */ + private Query buildBetweenQuery(String propertyName, Collection values) { + checkRequiredValuesSize(values, propertyName, "between", 2); + return Query.of(q -> q.range(getRangeQuery(propertyName, values, "between"))); + } + + /** + * Builds an existence query (exists, missing) + */ + private Query buildExistenceQuery(String propertyName, String operator) { + if (operator.equals("exists")) { + return Query.of(q -> q.exists(e -> e.field(propertyName))); + } else { // missing + return Query.of(q -> q.bool(b -> b.mustNot(m -> m.exists(e -> e.field(propertyName))))); + } + } + + /** + * Builds a content-based query (contains, startsWith, etc.) + */ + private Query buildContentQuery(String propertyName, String value, String operator) { + checkRequiredValue(value, propertyName, operator, false); + + return switch (operator) { + case "contains" -> Query.of(q -> q.regexp(r -> r.field(propertyName).value(".*" + value + ".*"))); + case "notContains" -> Query.of(q -> q.bool(b -> b.mustNot(m -> + m.regexp(r -> r.field(propertyName).value(".*" + value + ".*"))))); + case "startsWith" -> Query.of(q -> q.prefix(p -> p.field(propertyName).value(value))); + case "endsWith" -> Query.of(q -> q.regexp(r -> r.field(propertyName).value(".*" + value))); + case "matchesRegex" -> Query.of(q -> q.regexp(r -> r.field(propertyName).value(value))); + default -> throw new IllegalArgumentException("Unsupported content operator: " + operator); + }; + } + + /** + * Builds a collection-based query (in, notIn, all, etc.) + */ + private Query buildCollectionQuery(String propertyName, Collection values, String operator) { + checkRequiredValue(values, propertyName, operator, true); + + return switch (operator) { + case "in" -> Query.of(q -> q.terms(t -> t.field(propertyName).terms(t2 -> t2.value(getValues(values))))); + case "notIn" -> Query.of(q -> q.bool(b -> b.mustNot(m -> + m.terms(t -> t.field(propertyName).terms(t2 -> t2.value(getValues(values))))))); + case "all" -> { + BoolQuery.Builder all = new BoolQuery.Builder(); + for (Object curValue : values) { + all.must(Query.of(q -> q.term(t -> t.field(propertyName).value(getValue(curValue).build())))); + } + yield Query.of(q -> q.bool(all.build())); + } + case "inContains" -> { + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); + for (Object curValue : values) { + boolQueryBuilder.must(Query.of(q -> q.regexp(r -> r.field(propertyName).value(".*" + curValue + ".*")))); + } + yield Query.of(q -> q.bool(boolQueryBuilder.build())); + } + case "hasSomeOf" -> { + BoolQuery.Builder hasSomeOf = new BoolQuery.Builder(); + for (Object curValue : values) { + hasSomeOf.should(Query.of(q -> q.term(t -> t.field(propertyName).value(getValue(curValue).build())))); + } + yield Query.of(q -> q.bool(hasSomeOf.build())); + } + case "hasNoneOf" -> { + BoolQuery.Builder hasNoneOf = new BoolQuery.Builder(); + for (Object curValue : values) { + hasNoneOf.mustNot(Query.of(q -> q.term(t -> t.field(propertyName).value(getValue(curValue).build())))); + } + yield Query.of(q -> q.bool(hasNoneOf.build())); + } + default -> throw new IllegalArgumentException("Unsupported collection operator: " + operator); + }; + } + + /** + * Builds a date-based query (isDay, isNotDay) + */ + private Query buildDateQuery(String propertyName, Object value, String operator) { + checkRequiredValue(value, propertyName, operator, false); + if (operator.equals("isDay")) { + return getIsSameDayRange(getDate(value), propertyName); + } else { // isNotDay + return Query.of(q -> q.bool(b -> b.mustNot(getIsSameDayRange(getDate(value), propertyName)))); + } + } + + /** + * Builds a geographical distance query + */ + private Query buildDistanceQuery(Condition condition, String propertyName) { + final String unitString = (String) condition.getParameter("unit"); + final Object centerObj = condition.getParameter("center"); + final Double distance = (Double) condition.getParameter("distance"); + + if (centerObj == null || distance == null) { + throw new IllegalArgumentException("The 'center' and 'distance' parameters are required for the distance operator"); + } + + String centerString; + if (centerObj instanceof org.apache.unomi.api.GeoPoint) { + centerString = ((org.apache.unomi.api.GeoPoint) centerObj).asString(); + } else if (centerObj instanceof String) { + centerString = (String) centerObj; + } else { + centerString = centerObj.toString(); + } + + GeoDistanceType unit = unitString != null ? GeoDistanceType.valueOf(unitString) : GeoDistanceType.Plane; + + return Query.of(q -> q.geoDistance(g -> g.field(propertyName) + .distance(distance + "") + .distanceType(unit) + .location(l -> l.text(centerString)))); + } + + /** + * Creates a query to check if a date is on the same day + */ + private Query getIsSameDayRange(Date value, String name) { + DateTime date = new DateTime(value); + DateTime dayStart = date.withTimeAtStartOfDay(); + DateTime dayAfterStart = date.plusDays(1).withTimeAtStartOfDay(); + + return DateRangeQuery.of(d -> d.field(name) + .gte(dayStart.toString()) + .lt(dayAfterStart.toString())) + ._toRangeQuery() + ._toQuery(); + } + + /** + * Converts a date to ISO format + */ + private Object convertDateToISO(Object dateValue) { + if (dateValue == null) { + return null; + } + + if (dateValue instanceof Date) { + return dateTimeFormatter.print(new DateTime(dateValue)); + } else if (dateValue instanceof OffsetDateTime) { + return dateTimeFormatter.print(new DateTime(Date.from(((OffsetDateTime) dateValue).toInstant()))); + } else { + return dateValue; + } + } + + /** + * Converts a collection of dates to ISO format + */ + private Collection convertDatesToISO(Collection datesValues) { + if (datesValues == null) { + return null; + } + + List results = new ArrayList<>(datesValues.size()); + for (Object dateValue : datesValues) { + if (dateValue != null) { + results.add(convertDateToISO(dateValue)); + } + } + return results; + } + + /** + * Creates a range query based on the value type + */ + private RangeQuery getRangeQuery(String fieldName, Object value, String comparisonOperator) { + if (value instanceof String) { + return new RangeQuery.Builder() + .term(t -> withComparison(t.field(fieldName), (String) value, comparisonOperator)) + .build(); + } else if (value instanceof Date) { + return new RangeQuery.Builder() + .date(t -> withComparison(t.field(fieldName), convertDateToISO(value).toString(), comparisonOperator)) + .build(); + } else if (value instanceof Number) { + return new RangeQuery.Builder() + .number(t -> withComparison(t.field(fieldName), ((Number) value).doubleValue(), comparisonOperator)) + .build(); + } else if (value instanceof Collection) { + Iterator iterator = ((Collection) value).iterator(); + Object val1 = iterator.next(); + Object val2 = iterator.next(); + + if (val1 instanceof String) { + return new RangeQuery.Builder() + .term(t -> t.field(fieldName).gte((String) val1).lte((String) val2)) + .build(); + } else if (val1 instanceof Date) { + return new RangeQuery.Builder() + .date(t -> t.field(fieldName) + .gte(convertDateToISO(val1).toString()) + .lte(convertDateToISO(val2).toString())) + .build(); + } else if (val1 instanceof Number) { + return new RangeQuery.Builder() + .number(t -> t.field(fieldName) + .gte(((Number) val1).doubleValue()) + .lte(((Number) val2).doubleValue())) + .build(); + } + } + + throw new IllegalArgumentException("Unsupported value type for range query: " + + (value != null ? value.getClass().getName() : "null")); + } + + /** + * Applies the appropriate comparison operator to the range query builder + */ + private > T withComparison(RangeQueryBase.AbstractBuilder range, K value, String comparisonOperator) { + return switch (comparisonOperator) { + case "greaterThan" -> range.gt(value); + case "greaterThanOrEqualTo" -> range.gte(value); + case "lessThan" -> range.lt(value); + case "lessThanOrEqualTo" -> range.lte(value); + default -> throw new IllegalArgumentException("Unsupported comparison operator for range query: " + comparisonOperator); + }; + } + + /** + * Converts a value to Elasticsearch FieldValue + */ + private ObjectBuilder getValue(Object fieldValue) { + FieldValue.Builder fieldValueBuilder = new FieldValue.Builder(); + + if (fieldValue instanceof String) { + return fieldValueBuilder.stringValue((String) fieldValue); + } else if (fieldValue instanceof Integer) { + return fieldValueBuilder.longValue((Integer) fieldValue); + } else if (fieldValue instanceof Long) { + return fieldValueBuilder.longValue((Long) fieldValue); + } else if (fieldValue instanceof Double) { + return fieldValueBuilder.doubleValue((Double) fieldValue); + } else if (fieldValue instanceof Float) { + return fieldValueBuilder.doubleValue((Float) fieldValue); + } else if (fieldValue instanceof Boolean) { + return fieldValueBuilder.booleanValue((Boolean) fieldValue); + } else if (fieldValue instanceof Date || fieldValue instanceof OffsetDateTime) { + return fieldValueBuilder.stringValue(convertDateToISO(fieldValue).toString()); + } + + throw new IllegalArgumentException("Unsupported value type: " + + (fieldValue != null ? fieldValue.getClass().getName() : "null")); + } + + /** + * Converts a collection of values to a list of Elasticsearch FieldValues + */ + private List getValues(Collection fieldValues) { + List values = new ArrayList<>(fieldValues.size()); + for (Object fieldValue : fieldValues) { + values.add(getValue(fieldValue).build()); + } + return values; + } +} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/SourceEventPropertyConditionESQueryBuilder.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/SourceEventPropertyConditionESQueryBuilder.java new file mode 100644 index 0000000000..3534a62e07 --- /dev/null +++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/querybuilders/core/SourceEventPropertyConditionESQueryBuilder.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.unomi.persistence.elasticsearch.querybuilders.core; + +import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import org.apache.unomi.api.conditions.Condition; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class SourceEventPropertyConditionESQueryBuilder implements ConditionESQueryBuilder { + + public SourceEventPropertyConditionESQueryBuilder() { + } + + private void appendFilterIfPropExist(List queries, Condition condition, String prop) { + final Object parameter = condition.getParameter(prop); + if (parameter != null && !"".equals(parameter)) { + queries.add(Query.of(q -> q.term(t -> t.field("source." + prop).value(v -> v.stringValue((String) parameter))))); + } + } + + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + List queries = new ArrayList<>(); + for (String prop : new String[] { "id", "path", "scope", "type" }) { + appendFilterIfPropExist(queries, condition, prop); + } + + if (queries.isEmpty()) { + return null; + } else if (queries.size() == 1) { + return queries.get(0); + } else { + BoolQuery.Builder boolQueryBuilder = new BoolQuery.Builder(); + for (Query queryBuilder : queries) { + boolQueryBuilder.must(queryBuilder); + } + return Query.of(q -> q.bool(boolQueryBuilder.build())); + } + } +} diff --git a/persistence-elasticsearch/core/src/main/java/org/elasticsearch/client/CustomRestHighLevelClient.java b/persistence-elasticsearch/core/src/main/java/org/elasticsearch/client/CustomRestHighLevelClient.java deleted file mode 100644 index 8fff8dea67..0000000000 --- a/persistence-elasticsearch/core/src/main/java/org/elasticsearch/client/CustomRestHighLevelClient.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.elasticsearch.client; - -import org.elasticsearch.client.tasks.TaskSubmissionResponse; -import org.elasticsearch.index.reindex.DeleteByQueryRequest; -import org.elasticsearch.index.reindex.UpdateByQueryRequest; - -import java.io.IOException; - -import static java.util.Collections.emptySet; - -/** - * A custom Rest high level client that provide a way of using Task system on updateByQuery and deleteByQuery, - * by returning the response immediately (wait_for_completion set to false) - * see org.elasticsearch.client.RestHighLevelClient for original code. - */ -public class CustomRestHighLevelClient extends RestHighLevelClient { - - public CustomRestHighLevelClient(RestClientBuilder restClientBuilder) { - super(restClientBuilder); - } - - /** - * Executes a delete by query request. - * See - * Delete By Query API on elastic.co - * - * @param deleteByQueryRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - */ - public final TaskSubmissionResponse submitDeleteByQuery(DeleteByQueryRequest deleteByQueryRequest, RequestOptions options) throws IOException { - return performRequestAndParseEntity( - deleteByQueryRequest, innerDeleteByQueryRequest -> { - Request request = RequestConverters.deleteByQuery(innerDeleteByQueryRequest); - request.addParameter("wait_for_completion", "false"); - return request; - }, options, TaskSubmissionResponse::fromXContent, emptySet() - ); - } - - /** - * Executes a update by query request. - * See - * Update By Query API on elastic.co - * - * @param updateByQueryRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - */ - public final TaskSubmissionResponse submitUpdateByQuery(UpdateByQueryRequest updateByQueryRequest, RequestOptions options) throws IOException { - return performRequestAndParseEntity( - updateByQueryRequest, innerUpdateByQueryRequest -> { - Request request = RequestConverters.updateByQuery(updateByQueryRequest); - request.addParameter("wait_for_completion", "false"); - return request; - }, options, TaskSubmissionResponse::fromXContent, emptySet() - ); - } -} diff --git a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml index b9608f8581..e5e9ee0ad4 100644 --- a/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/persistence-elasticsearch/core/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -23,7 +23,7 @@ http://aries.apache.org/blueprint/xmlns/blueprint-cm/v1.3.0 https://aries.apache.org/schemas/blueprint-cm/blueprint-cm-1.3.0.xsd"> + update-strategy="reload" placeholder-prefix="${es."> @@ -32,75 +32,61 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + + org.apache.unomi.persistence.spi.PersistenceService org.osgi.framework.SynchronousBundleListener - org.osgi.service.cm.ManagedService - - - - - - - - - + class="org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher"> + - + - - - - - @@ -123,64 +104,113 @@ - - - - - - - - - - - + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + bind-method="bindConditionEvaluator" unbind-method="unbindConditionEvaluator" + ref="elasticSearchPersistenceServiceImpl"/> + interface="org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder" + availability="optional"> + bind-method="bindConditionESQueryBuilder" + unbind-method="unbindConditionESQueryBuilder" + ref="elasticSearchPersistenceServiceImpl"/> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg index 224d01110c..ff1fbfb1b6 100644 --- a/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg +++ b/persistence-elasticsearch/core/src/main/resources/org.apache.unomi.persistence.elasticsearch.cfg @@ -15,7 +15,7 @@ # limitations under the License. # -cluster.name=${org.apache.unomi.elasticsearch.cluster.name:-contextElasticSearch} +cluster.name=${org.apache.unomi.elasticsearch.cluster.name:-elasticsearch-cluster} # The elasticSearchAddresses may be a comma seperated list of host names and ports such as # hostA:9200,hostB:9200 # Note: the port number must be repeated for each host. @@ -23,18 +23,12 @@ elasticSearchAddresses=${org.apache.unomi.elasticsearch.addresses:-localhost:920 fatalIllegalStateErrors=${org.apache.unomi.elasticsearch.fatalIllegalStateErrors:-} index.prefix=${org.apache.unomi.elasticsearch.index.prefix:-context} -# Deprecated properties. Please use rollover corresponding properties -monthlyIndex.numberOfShards=${org.apache.unomi.elasticsearch.monthlyIndex.nbShards:-5} -monthlyIndex.numberOfReplicas=${org.apache.unomi.elasticsearch.monthlyIndex.nbReplicas:-0} -monthlyIndex.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.monthlyIndex.indexMappingTotalFieldsLimit:-1000} -monthlyIndex.indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.monthlyIndex.indexMaxDocValueFieldsSearch:-1000} -monthlyIndex.itemsMonthlyIndexedOverride=${org.apache.unomi.elasticsearch.monthlyIndex.itemsMonthlyIndexedOverride:-event,session} -# New properties for index rotation: -rollover.numberOfShards=${org.apache.unomi.elasticsearch.rollover.nbShards} -rollover.numberOfReplicas=${org.apache.unomi.elasticsearch.rollover.nbReplicas} -rollover.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit} -rollover.indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch} -rollover.indices=${org.apache.unomi.elasticsearch.rollover.indices} +# Properties for index rotation: +rollover.numberOfShards=${org.apache.unomi.elasticsearch.rollover.nbShards:-5} +rollover.numberOfReplicas=${org.apache.unomi.elasticsearch.rollover.nbReplicas:-0} +rollover.indexMappingTotalFieldsLimit=${org.apache.unomi.elasticsearch.rollover.indexMappingTotalFieldsLimit:-1000} +rollover.indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.rollover.indexMaxDocValueFieldsSearch:-1000} +rollover.indices=${org.apache.unomi.elasticsearch.rollover.indices:-event,session} numberOfShards=${org.apache.unomi.elasticsearch.defaultIndex.nbShards:-5} numberOfReplicas=${org.apache.unomi.elasticsearch.defaultIndex.nbReplicas:-0} @@ -43,7 +37,7 @@ indexMaxDocValueFieldsSearch=${org.apache.unomi.elasticsearch.defaultIndex.index defaultQueryLimit=${org.apache.unomi.elasticsearch.defaultQueryLimit:-10} # Rollover amd index configuration for event and session indices, values are cumulative -# See https://www.elastic.co/guide/en/elasticsearch/reference/7.17/ilm-rollover.html for option details. +# See https://www.elastic.co/docs/reference/elasticsearch/index-lifecycle-actions/ilm-rollover for option details. rollover.maxSize=${org.apache.unomi.elasticsearch.rollover.maxSize:-30gb} rollover.maxAge=${org.apache.unomi.elasticsearch.rollover.maxAge} rollover.maxDocs=${org.apache.unomi.elasticsearch.rollover.maxDocs} @@ -53,16 +47,16 @@ rollover.maxDocs=${org.apache.unomi.elasticsearch.rollover.maxDocs} # The values used here are the default values of the API bulkProcessor.concurrentRequests=${org.apache.unomi.elasticsearch.bulkProcessor.concurrentRequests:-1} bulkProcessor.bulkActions=${org.apache.unomi.elasticsearch.bulkProcessor.bulkActions:-1000} -bulkProcessor.bulkSize=${org.apache.unomi.elasticsearch.bulkProcessor.bulkSize:-5MB} -bulkProcessor.flushInterval=${org.apache.unomi.elasticsearch.bulkProcessor.flushInterval:-5s} +bulkProcessor.bulkSize=${org.apache.unomi.elasticsearch.bulkProcessor.bulkSize:-5} +bulkProcessor.flushInterval=${org.apache.unomi.elasticsearch.bulkProcessor.flushInterval:-5} bulkProcessor.backoffPolicy=${org.apache.unomi.elasticsearch.bulkProcessor.backoffPolicy:-exponential} -# The following settings are used to perform version checks on the connected ElasticSearch cluster, to make sure that +# The following settings are used to perform version checks on the connected Elasticsearch cluster, to make sure that # appropriate versions are used. The check is performed like this : -# for each node in the ElasticSearch cluster: -# minimalElasticSearchVersion <= ElasticSearch node version < maximalElasticSearchVersion -minimalElasticSearchVersion=7.0.0 -maximalElasticSearchVersion=8.0.0 +# for each node in the Elasticsearch cluster: +# minimalElasticsearchVersion <= Elasticsearch node version < maximalElasticsearchVersion +minimalElasticsearchVersion=9.0.3 +maximalElasticsearchVersion=10.0.0 # The following setting is used to set the aggregate query bucket size aggregateQueryBucketSize=${org.apache.unomi.elasticsearch.aggregateQueryBucketSize:-5000} @@ -79,14 +73,14 @@ pastEventsDisablePartitions=${org.apache.unomi.elasticsearch.pastEventsDisablePa clientSocketTimeout=${org.apache.unomi.elasticsearch.clientSocketTimeout:--1} # Defines the waiting for task completion timeout in milliseconds. -# Some operations like update_by_query and delete_by_query are delegated to ElasticSearch using tasks -# For consistency the thread that trigger one of those operations will wait for the task to be completed on ElasticSearch side. +# Some operations like update_by_query and delete_by_query are delegated to Elasticsearch using tasks +# For consistency the thread that trigger one of those operations will wait for the task to be completed on Elasticsearch side. # This timeout configuration is here to ensure not blocking the thread infinitely, in case of very long running tasks. # A timeout value of zero or negative is interpreted as an infinite timeout. # Default: 3600000 (1 hour) taskWaitingTimeout=${org.apache.unomi.elasticsearch.taskWaitingTimeout:-3600000} -# Defines the polling interval in milliseconds, which is used to check if task is completed on ElasticSearch side +# Defines the polling interval in milliseconds, which is used to check if task is completed on Elasticsearch side # Default: 1000 (1 second) taskWaitingPollingInterval=${org.apache.unomi.elasticsearch.taskWaitingPollingInterval:-1000} @@ -103,8 +97,8 @@ aggQueryMaxResponseSizeHttp=${org.apache.unomi.elasticsearch.aggQueryMaxResponse # Authentication username=${org.apache.unomi.elasticsearch.username:-} password=${org.apache.unomi.elasticsearch.password:-} -sslEnable=${org.apache.unomi.elasticsearch.sslEnable:-false} -sslTrustAllCertificates=${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-false} +sslEnable=${org.apache.unomi.elasticsearch.sslEnable:-true} +sslTrustAllCertificates=${org.apache.unomi.elasticsearch.sslTrustAllCertificates:-true} # Errors throwExceptions=${org.apache.unomi.elasticsearch.throwExceptions:-false} @@ -114,3 +108,5 @@ useBatchingForUpdate=${org.apache.unomi.elasticsearch.useBatchingForUpdate:-true # ES logging logLevelRestClient=${org.apache.unomi.elasticsearch.logLevelRestClient:-ERROR} + +minimalClusterState=${org.apache.unomi.elasticsearch.minimalClusterState:-GREEN} diff --git a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/ElasticsearchPersistenceTest.java b/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/ElasticsearchPersistenceTest.java deleted file mode 100644 index 9c65328d88..0000000000 --- a/persistence-elasticsearch/core/src/test/java/org/apache/unomi/persistence/elasticsearch/ElasticsearchPersistenceTest.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.unomi.persistence.elasticsearch; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import org.apache.http.HttpHost; -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.index.IndexResponse; -import org.elasticsearch.client.RequestOptions; -import org.elasticsearch.client.RestClient; -import org.elasticsearch.client.RestHighLevelClient; -import org.elasticsearch.client.core.MainResponse; -import org.elasticsearch.client.indices.CreateIndexRequest; -import org.elasticsearch.client.indices.CreateIndexResponse; -import org.elasticsearch.cluster.ClusterName; -import org.elasticsearch.common.network.NetworkModule; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.core.internal.io.IOUtils; -import org.elasticsearch.env.Environment; -import org.elasticsearch.node.MockNode; -import org.elasticsearch.node.Node; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.transport.Netty4Plugin; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.junit.runner.RunWith; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Date; -import java.util.UUID; -import java.util.logging.Logger; - -@RunWith(com.carrotsearch.randomizedtesting.RandomizedRunner.class) -@ThreadLeakScope(value = ThreadLeakScope.Scope.NONE) -public class ElasticsearchPersistenceTest { - - private static final Logger LOGGER = Logger.getLogger(ElasticsearchPersistenceTest.class.getName()); - - private static final String CLUSTER_NAME = "unomi-cluster-test"; - private static final String NODE_NAME = "unomi-node-test"; - private static final String HOST = "127.0.0.1"; - private static final int HTTP_PORT_NODE_1 = 9200+10; - private static final int HTTP_PORT_NODE_2 = 9201+10; - private static final int TRANSPORT_PORT_NODE_1 = 9300+10; - private static final int TRANSPORT_PORT_NODE_2 = 9301+10; - - private static RestHighLevelClient restHighLevelClient; - - private static Node node1; - private static Node node2; - - @BeforeClass - public static void setup() throws Exception { - Collection plugins = Arrays.asList(Netty4Plugin.class); - - Settings settingsNode1 = Settings.builder() - .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), CLUSTER_NAME) - .put(Node.NODE_NAME_SETTING.getKey(), NODE_NAME + "-1") - .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) - .put(Environment.PATH_HOME_SETTING.getKey(), "target/data-1") - .put(Environment.PATH_DATA_SETTING.getKey(), "target/data-1") - .put("network.host", HOST) - .put("http.port", HTTP_PORT_NODE_1) - .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) - .put("transport.port", TRANSPORT_PORT_NODE_1) - .build(); - node1 = new MockNode(settingsNode1, plugins); - node1.start(); - - Settings settingsNode2 = Settings.builder() - .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), CLUSTER_NAME) - .put(Node.NODE_NAME_SETTING.getKey(), NODE_NAME + "-2") - .put(NetworkModule.HTTP_TYPE_KEY, Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME) - .put(Environment.PATH_HOME_SETTING.getKey(), "target/data-2") - .put(Environment.PATH_DATA_SETTING.getKey(), "target/data-2") - .put("network.host", HOST) - .put("http.port", HTTP_PORT_NODE_2) - .put(NetworkModule.TRANSPORT_TYPE_KEY, Netty4Plugin.NETTY_TRANSPORT_NAME) - .put("transport.port", TRANSPORT_PORT_NODE_2) - .build(); - node2 = new MockNode(settingsNode2, plugins); - node2.start(); - - restHighLevelClient = new RestHighLevelClient(RestClient.builder( - new HttpHost(HOST, HTTP_PORT_NODE_1, "http"), - new HttpHost(HOST, HTTP_PORT_NODE_2, "http"))); - } - - @AfterClass - public static void teardown() throws Exception { - IOUtils.close(restHighLevelClient); - if (node1 != null) { - node1.close(); - } - if (node2 != null) { - node2.close(); - } - } - - @Test - public void testGetClusterInfo() throws Exception { - MainResponse response = restHighLevelClient.info(RequestOptions.DEFAULT); - LOGGER.info("Cluster getMinimumIndexCompatibilityVersion: " + response.getVersion().getMinimumIndexCompatibilityVersion()); - LOGGER.info("Cluster getMinimumWireCompatibilityVersion: " + response.getVersion().getMinimumWireCompatibilityVersion()); - LOGGER.info("Cluster number: " + response.getVersion().getNumber()); - } - - @Test - public void testCreateIndex() throws Exception { - restHighLevelClient.info(RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build()); - final String indexName = "unomi-index-" + new Date().getTime(); - CreateIndexRequest request = new CreateIndexRequest(indexName); - CreateIndexResponse response = restHighLevelClient.indices().create(request, RequestOptions.DEFAULT); - if (response.isAcknowledged()) { - LOGGER.info(">>> Create index :: ok :: name = " + response.index()); - } else { - LOGGER.info(">>> Create index :: not acknowledged"); - } - -// ClusterHealthResponse actionGet = restHighLevelClient.cluster() -// .health(Requests.clusterHealthRequest("unomi-index-1").waitForGreenStatus().waitForEvents(Priority.LANGUID) -// .waitForNoRelocatingShards(true), RequestOptions.DEFAULT); -// Assert.assertNotNull(actionGet); -// -// switch (actionGet.getStatus()) { -// case GREEN: -// logger.info(">>> Cluster State :: GREEN"); -// break; -// case YELLOW: -// logger.info(">>> Cluster State :: YELLOW"); -// break; -// case RED: -// logger.info(">>> Cluster State :: RED"); -// break; -// } -// Assert.assertNotEquals(actionGet.getStatus(), ClusterHealthStatus.RED); - - IndexRequest indexRequest = new IndexRequest(indexName); - indexRequest.id(UUID.randomUUID().toString()); - String type = "{\"type\":\"unomi-type\"}"; - String source = "{\"name\":\"unomi-name\"}"; - indexRequest.source(XContentType.JSON, type, source); - ActionRequestValidationException exception = indexRequest.validate(); - Assert.assertNull(exception); - - IndexResponse indexResponse = restHighLevelClient.index(indexRequest, RequestOptions.DEFAULT); - Assert.assertNotNull(indexResponse); - if (indexResponse.status() == RestStatus.CREATED) { - LOGGER.info(">>> Insert data created"); - } else { - LOGGER.info(">>> Insert data ko :: " + indexResponse.status().name()); - } - Assert.assertEquals(indexResponse.status(), RestStatus.CREATED); - } - -} diff --git a/persistence-elasticsearch/pom.xml b/persistence-elasticsearch/pom.xml index 0ac011e47e..78df1dca4f 100644 --- a/persistence-elasticsearch/pom.xml +++ b/persistence-elasticsearch/pom.xml @@ -26,12 +26,13 @@ unomi-persistence-elasticsearch - Apache Unomi :: Persistence :: ElasticSearch - ElasticSearch persistence implementation for the Apache Unomi Context Server + Apache Unomi :: Persistence :: Elasticsearch + Elasticsearch persistence implementation for the Apache Unomi Context Server pom core + conditions @@ -44,6 +45,12 @@ *;scope=compile|runtime + org.apache.unomi.api.conditions, + org.apache.unomi.api.query, + org.apache.unomi.api, + org.apache.unomi.metrics, + org.apache.unomi.persistence.spi.aggregate, + org.apache.unomi.persistence.spi, com.conversantmedia.util.concurrent;resolution:=optional, * diff --git a/persistence-spi/pom.xml b/persistence-spi/pom.xml index 4815944bf6..f33e066656 100644 --- a/persistence-spi/pom.xml +++ b/persistence-spi/pom.xml @@ -16,7 +16,8 @@ ~ limitations under the License. --> - + 4.0.0 org.apache.unomi @@ -46,6 +47,16 @@ unomi-api provided + + org.apache.unomi + unomi-scripting + provided + + + org.apache.unomi + unomi-metrics + provided + com.fasterxml.jackson.core jackson-core @@ -76,12 +87,16 @@ commons-collections provided + + org.apache.commons + commons-lang3 + provided + org.slf4j slf4j-api provided - junit @@ -93,6 +108,27 @@ slf4j-simple test + + commons-io + commons-io + + + + + org.apache.felix + maven-bundle-plugin + true + + + *;scope=compile|runtime + + * + + + + + + diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java index 0fe3746161..964957e537 100644 --- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java @@ -27,13 +27,18 @@ import java.util.Date; import java.util.List; import java.util.Map; -import java.util.function.Consumer; /** * A service to provide persistence and retrieval of context server entities. */ public interface PersistenceService { + /** + * A unique name to identify the persistence service. + * @return a string containing the unique name for the persistence service. + */ + String getName(); + /** * Retrieves all known items of the specified class. * WARNING: this method can be quite computationally intensive and calling the paged version {@link #getAllItems(Class, int, int, String)} is preferred. @@ -338,23 +343,6 @@ default CustomItem loadCustomItem(String itemId, String customItemType) { */ boolean removeByQuery(Condition query, Class clazz); - /** - * Persists the specified query under the specified name. - * - * @param queryName the name under which the specified query should be recorded - * @param query the query to be recorded - * @return {@code true} if the query was properly saved, {@code false} otherwise - */ - boolean saveQuery(String queryName, Condition query); - - /** - * Deletes the query identified by the specified name. - * - * @param queryName the name under which the specified query was recorded - * @return {@code true} if the deletion was successful, {@code false} otherwise - */ - boolean removeQuery(String queryName); - /** * Retrieve the type mappings for a given itemType. This method queries the persistence service implementation * to retrieve any type mappings it may have for the specified itemType. @@ -705,24 +693,6 @@ default void refreshIndex(Class clazz) { */ void purgeTimeBasedItems(int existsNumberOfDays, Class clazz); - /** - * Retrieves all items of the specified Item subclass which specified ranged property is within the specified bounds, ordered according to the specified {@code sortBy} String - * and and paged: only {@code size} of them are retrieved, starting with the {@code offset}-th one. - * - * @param the type of the Item subclass we want to retrieve - * @param s the name of the range property we want items to retrieve to be included between the specified start and end points - * @param from the beginning of the range we want to consider - * @param to the end of the range we want to consider - * @param sortBy an optional ({@code null} if no sorting is required) String of comma ({@code ,}) separated property names on which ordering should be performed, ordering - * elements according to the property order in the String, considering each in turn and moving on to the next one in case of equality of all preceding ones. - * Each property name is optionally followed by a column ({@code :}) and an order specifier: {@code asc} or {@code desc}. - * @param clazz the {@link Item} subclass of the items we want to retrieve - * @param offset zero or a positive integer specifying the position of the first item in the total ordered collection of matching items - * @param size a positive integer specifying how many matching items should be retrieved or {@code -1} if all of them should be retrieved - * @return a {@link PartialList} of items matching the specified criteria - */ - PartialList rangeQuery(String s, String from, String to, String sortBy, Class clazz, int offset, int size); - /** * Retrieves the specified metrics for the specified field of items of the specified type as defined by the Item subclass public field {@code ITEM_TYPE} and matching the * specified {@link Condition}. diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/aggregate/DateAggregate.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/aggregate/DateAggregate.java index 037727a683..9eca674c81 100644 --- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/aggregate/DateAggregate.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/aggregate/DateAggregate.java @@ -17,54 +17,123 @@ package org.apache.unomi.persistence.spi.aggregate; -public class DateAggregate extends BaseAggregate{ - public static final DateAggregate SECOND = new DateAggregate("1s"); - public static final DateAggregate MINUTE = new DateAggregate("1m"); - public static final DateAggregate HOUR = new DateAggregate("1h"); - public static final DateAggregate DAY = new DateAggregate("1d"); - public static final DateAggregate WEEK = new DateAggregate("1w"); - public static final DateAggregate MONTH = new DateAggregate("1M"); - public static final DateAggregate QUARTER = new DateAggregate("1q"); - public static final DateAggregate YEAR = new DateAggregate("1y"); +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class DateAggregate extends BaseAggregate { private static final String DEFAULT_INTERVAL = "1M"; + private String interval; private String format; + + // Maps bidirectionnelles pour la conversion entre formats + private static final Map OLD_TO_NEW_FORMAT = Map.ofEntries( + Map.entry("1s", "Second"), + Map.entry("1m", "Minute"), + Map.entry("1h", "Hour"), + Map.entry("1d", "Day"), + Map.entry("1w", "Week"), + Map.entry("1M", "Month"), + Map.entry("1q", "Quarter"), + Map.entry("1y", "Year") + ); + + private static final Map NEW_TO_OLD_FORMAT = createReverseMap(); + + private static Map createReverseMap() { + Map reverseMap = new HashMap<>(); + for (Map.Entry entry : DateAggregate.OLD_TO_NEW_FORMAT.entrySet()) { + reverseMap.put(entry.getValue(), entry.getKey()); + } + return Collections.unmodifiableMap(reverseMap); + } + public DateAggregate(String field) { super(field); this.interval = DEFAULT_INTERVAL; } + public DateAggregate(String field, String interval) { super(field); - this.interval = (interval != null && interval.length() > 0) ? interval : DEFAULT_INTERVAL; + setInterval(interval); } + public DateAggregate(String field, String interval, String format) { super(field); - this.interval = (interval != null && interval.length() > 0) ? interval : DEFAULT_INTERVAL; + setInterval(interval); this.format = format; } - public static DateAggregate seconds(int sec) { - return new DateAggregate(sec + "s"); + public void setInterval(String interval) { + this.interval = (interval != null && !interval.isEmpty()) ? interval : DEFAULT_INTERVAL; } - public static DateAggregate minutes(int min) { - return new DateAggregate(min + "m"); + /** + * Returns the interval as it was originally defined + */ + public String getInterval() { + return interval; } - public static DateAggregate hours(int hours) { - return new DateAggregate(hours + "h"); + /** + * Returns the interval in the old format (1M, 1d, etc.) + */ + public String getIntervalInOldFormat() { + if (isOldFormat(interval)) { + return interval; + } + return NEW_TO_OLD_FORMAT.getOrDefault(interval, interval); } - public static DateAggregate days(int days) { - return new DateAggregate(days + "d"); + /** + * Returns the interval in the new format (Month, Day, etc.) + */ + public String getIntervalInNewFormat() { + if (isNewFormat(interval)) { + return interval; + } + return OLD_TO_NEW_FORMAT.getOrDefault(interval, interval); } - public static DateAggregate weeks(int weeks) { - return new DateAggregate(weeks + "w"); + /** + * Compatibility method with old code + * @deprecated Use getIntervalInNewFormat() instead + */ + @Deprecated + public String getIntervalByAlias(String alias) { + if (isOldFormat(alias)) { + return OLD_TO_NEW_FORMAT.getOrDefault(alias, alias); + } + return alias; } - public String getInterval() { - return interval; + /** + * Determines if the interval uses the old format + */ + public boolean isOldFormat(String value) { + return OLD_TO_NEW_FORMAT.containsKey(value); + } + + /** + * Determines if the interval uses the new format + */ + public boolean isNewFormat(String value) { + return NEW_TO_OLD_FORMAT.containsKey(value); + } + + /** + * Converts from old format to new format + */ + public static String convertToNewFormat(String oldFormat) { + return OLD_TO_NEW_FORMAT.getOrDefault(oldFormat, oldFormat); + } + + /** + * Converts from new format to old format + */ + public static String convertToOldFormat(String newFormat) { + return NEW_TO_OLD_FORMAT.getOrDefault(newFormat, newFormat); } public String getFormat() { @@ -74,4 +143,4 @@ public String getFormat() { public void setFormat(String format) { this.format = format; } -} +} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionContextHelper.java similarity index 62% rename from persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java rename to persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionContextHelper.java index 5aae455ec3..35dbfcdec8 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionContextHelper.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionContextHelper.java @@ -15,35 +15,70 @@ * limitations under the License. */ -package org.apache.unomi.persistence.elasticsearch.conditions; +package org.apache.unomi.persistence.spi.conditions; import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.core.util.IOUtils; -import org.apache.lucene.analysis.charfilter.MappingCharFilterFactory; -import org.apache.lucene.analysis.util.ClasspathResourceLoader; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.scripting.ScriptExecutor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; + +import java.io.IOException; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; + public class ConditionContextHelper { private static final Logger LOGGER = LoggerFactory.getLogger(ConditionContextHelper.class); - private static MappingCharFilterFactory mappingCharFilterFactory; + private static final Map FOLD_MAPPING = new HashMap<>(); + static { - Map args = new HashMap<>(); - args.put("mapping", "mapping-FoldToASCII.txt"); - mappingCharFilterFactory = new MappingCharFilterFactory(args); try { - mappingCharFilterFactory.inform(new ClasspathResourceLoader(ConditionContextHelper.class.getClassLoader())); + loadMappingFile(); } catch (IOException e) { - e.printStackTrace(); + LOGGER.error("Erreur lors du chargement du fichier de mapping", e); + } + } + + private static void loadMappingFile() throws IOException { + try (InputStream is = ConditionContextHelper.class.getClassLoader().getResourceAsStream("mapping-FoldToASCII.txt"); + BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + + String line; + while ((line = reader.readLine()) != null) { + if (line.trim().isEmpty() || line.startsWith("#")) { + continue; + } + + if (line.contains("=>")) { + String[] parts = line.split("=>"); + if (parts.length == 2) { + String unicodeStr = parts[0].trim(); + String asciiStr = parts[1].trim(); + + if (unicodeStr.startsWith("\"\\u") && unicodeStr.endsWith("\"")) { + String hexCode = unicodeStr.substring(3, unicodeStr.length() - 1); + try { + char unicodeChar = (char) Integer.parseInt(hexCode, 16); + + if (asciiStr.startsWith("\"") && asciiStr.endsWith("\"")) { + String asciiValue = asciiStr.substring(1, asciiStr.length() - 1); + FOLD_MAPPING.put(unicodeChar, asciiValue); + } + } catch (NumberFormatException e) { + LOGGER.warn("Format de code Unicode invalide: {}", hexCode); + } + } + } + } + } } } @@ -117,6 +152,20 @@ private static boolean hasContextualParameter(Object value) { return false; } + public static String forceFoldToASCII(Object object) { + if (object != null) { + return foldToASCII(object.toString()); + } + return null; + } + + public static Collection forceFoldToASCII(Collection collection) { + if (collection != null) { + return collection.stream().map(ConditionContextHelper::forceFoldToASCII).collect(Collectors.toList()); + } + return null; + } + public static String[] foldToASCII(String[] s) { if (s != null) { for (int i = 0; i < s.length; i++) { @@ -127,15 +176,25 @@ public static String[] foldToASCII(String[] s) { } public static String foldToASCII(String s) { - if (s != null) { - s = s.toLowerCase(); - try (StringReader stringReader = new StringReader(s); Reader foldedStringReader = mappingCharFilterFactory.create(stringReader)) { - return IOUtils.toString(foldedStringReader); - } catch (IOException e) { - LOGGER.error("Error folding to ASCII string {}", s, e); + if (s == null) { + return null; + } + + s = s.toLowerCase(); + StringBuilder result = new StringBuilder(s.length()); + + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + String mapped = FOLD_MAPPING.get(c); + + if (mapped != null) { + result.append(mapped); + } else { + result.append(c); } } - return null; + + return result.toString(); } public static Collection foldToASCII(Collection s) { @@ -150,4 +209,4 @@ public static Collection foldToASCII(Collection s) { return null; } -} +} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluator.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluator.java similarity index 94% rename from persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluator.java rename to persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluator.java index 86f8f52d62..97af4ed97a 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluator.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluator.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.unomi.persistence.elasticsearch.conditions; +package org.apache.unomi.persistence.spi.conditions; import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluatorDispatcher.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluatorDispatcher.java new file mode 100644 index 0000000000..0c6f438b4c --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluatorDispatcher.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions; + +import org.apache.unomi.api.Item; +import org.apache.unomi.api.conditions.Condition; + +import java.util.Map; + +public interface ConditionEvaluatorDispatcher { + + void addEvaluator(String name, ConditionEvaluator evaluator); + + void removeEvaluator(String name); + + boolean eval(Condition condition, Item item); + + boolean eval(Condition condition, Item item, Map context); +} \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluatorDispatcherImpl.java similarity index 91% rename from persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java rename to persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluatorDispatcherImpl.java index 48e54e00aa..5a7f4cfc32 100644 --- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/conditions/ConditionEvaluatorDispatcher.java +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/ConditionEvaluatorDispatcherImpl.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.unomi.persistence.elasticsearch.conditions; +package org.apache.unomi.persistence.spi.conditions; import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; @@ -32,7 +32,8 @@ /** * Entry point for condition evaluation. Will dispatch to all evaluators. */ -public class ConditionEvaluatorDispatcher { +//TODO change to delarative services remove blueprint +public class ConditionEvaluatorDispatcherImpl implements ConditionEvaluatorDispatcher { private static final Logger LOGGER = LoggerFactory.getLogger(ConditionEvaluatorDispatcher.class.getName()); private Map evaluators = new ConcurrentHashMap<>(); @@ -40,6 +41,8 @@ public class ConditionEvaluatorDispatcher { private MetricsService metricsService; private ScriptExecutor scriptExecutor; + public ConditionEvaluatorDispatcherImpl() {} + public void setMetricsService(MetricsService metricsService) { this.metricsService = metricsService; } @@ -48,18 +51,22 @@ public void setScriptExecutor(ScriptExecutor scriptExecutor) { this.scriptExecutor = scriptExecutor; } + @Override public void addEvaluator(String name, ConditionEvaluator evaluator) { evaluators.put(name, evaluator); } + @Override public void removeEvaluator(String name) { evaluators.remove(name); } + @Override public boolean eval(Condition condition, Item item) { - return eval(condition, item, new HashMap()); + return eval(condition, item, new HashMap<>()); } + @Override public boolean eval(Condition condition, Item item, Map context) { String conditionEvaluatorKey = condition.getConditionType().getConditionEvaluator(); if (condition.getConditionType().getParentCondition() != null) { diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/DateUtils.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/DateUtils.java new file mode 100644 index 0000000000..db6f773e2e --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/DateUtils.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions; + +import org.apache.unomi.persistence.spi.conditions.datemath.DateMathParseException; +import org.apache.unomi.persistence.spi.conditions.datemath.DateMathParser; +import org.apache.unomi.persistence.spi.conditions.datemath.JavaDateFormatter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Date; + +public class DateUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(DateUtils.class.getName()); + + public static Date getDate(Object value) { + if (value == null) { + return null; + } + if (value instanceof Date) { + return (Date) value; + } else { + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_optional_time||epoch_millis"); + DateMathParser dateMathParser = new DateMathParser(formatter, DateTimeFormatter.ISO_DATE_TIME); + try { + Instant instant = dateMathParser.parse(value.toString(), System::currentTimeMillis, false, ZoneOffset.UTC); + return Date.from(instant); + } catch (DateMathParseException e) { + LOGGER.warn("unable to parse date. See debug log level for full stacktrace"); + LOGGER.warn("unable to parse date {}", value, e); + } + return null; + } + } +} \ No newline at end of file diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/PastEventConditionPersistenceQueryBuilder.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/PastEventConditionPersistenceQueryBuilder.java new file mode 100644 index 0000000000..0379c9711a --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/PastEventConditionPersistenceQueryBuilder.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions; + +import org.apache.unomi.api.conditions.Condition; +import org.apache.unomi.api.services.DefinitionsService; +import org.apache.unomi.scripting.ScriptExecutor; + +import java.util.Map; + +public interface PastEventConditionPersistenceQueryBuilder { + + boolean getStrategyFromOperator(String operator); + + Condition getEventCondition(Condition condition, Map context, String profileId, + DefinitionsService definitionsService, ScriptExecutor scriptExecutor); +} diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/DateMathParseException.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/DateMathParseException.java new file mode 100644 index 0000000000..60f5af9be4 --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/DateMathParseException.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions.datemath; + +/** + * Exception thrown by the {@link DateMathParser} when a malformed date math expression is encountered. + */ +public class DateMathParseException extends RuntimeException { + public DateMathParseException(String message) { + super(message); + } + + public DateMathParseException(String message, Throwable cause) { + super(message, cause); + } + + public DateMathParseException(String message, Object... args) { + super(String.format(message, args)); + } + + public DateMathParseException(String message, Throwable cause, Object... args) { + super(String.format(message, args), cause); + } +} diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/DateMathParser.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/DateMathParser.java new file mode 100644 index 0000000000..4c599c1c9d --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/DateMathParser.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions.datemath; + +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalAdjusters; +import java.time.temporal.TemporalQueries; +import java.util.function.Function; +import java.util.function.LongSupplier; + +public class DateMathParser { + + public static boolean isNullOrEmpty(CharSequence cs) { + return cs == null || cs.length() == 0; + } + + public static class DateFormatters { + public static ZonedDateTime from(TemporalAccessor accessor) { + // Default to UTC if no zone or offset is present + ZoneId zone = accessor.query(TemporalQueries.zone()); + if (zone == null) { + ZoneOffset offset = accessor.query(TemporalQueries.offset()); + zone = (offset != null) ? offset : ZoneOffset.UTC; + } + + // If the accessor supports INSTANT_SECONDS, construct from Instant + if (accessor.isSupported(ChronoField.INSTANT_SECONDS)) { + return ZonedDateTime.ofInstant(Instant.from(accessor), zone); + } + + // Handle LocalDate and LocalTime explicitly + LocalDate date = accessor.query(TemporalQueries.localDate()); + LocalTime time = accessor.query(TemporalQueries.localTime()); + + // Ensure missing components are handled gracefully + if (date == null && time == null) { + throw new DateTimeException("Cannot extract LocalDate or LocalTime from TemporalAccessor"); + } + + if (date == null) { + date = LocalDate.ofEpochDay(0); // Default to 1970-01-01 + } + + if (time == null) { + time = LocalTime.MIDNIGHT; // Default to 00:00 + } + + // Combine LocalDate and LocalTime with ZoneId + return ZonedDateTime.of(date, time, zone); + } + } + + private final JavaDateFormatter formatter; + private final DateTimeFormatter roundUpFormatter; + + public DateMathParser(JavaDateFormatter formatter, DateTimeFormatter roundUpFormatter) { + this.formatter = formatter; + this.roundUpFormatter = roundUpFormatter; + } + + private String normalizeDateMathInput(String input) { + // Replace 't' with 'T' only when it's part of an ISO datetime format (e.g., `2022-05-18t15:23:17z`) + input = input.replaceAll("(?<=\\d{4}-\\d{2}-\\d{2})t", "T"); // Match 't' after a full date + // Replace 'z' with 'Z' only when it's at the end of the string or follows time components + input = input.replaceAll("z$", "Z"); // Match 'z' at the end + input = input.replaceAll("(?<=[:\\d])z", "Z"); // Match 'z' after a time component + return input; + } + + public Instant parse(String text, LongSupplier now, boolean roundUpProperty, ZoneId timeZone) { + text = text.trim(); + + Instant time; + String mathString; + if (text.startsWith("now")) { + try { + time = Instant.ofEpochMilli(now.getAsLong()); + } catch (Exception e) { + throw new DateMathParseException("could not read the current timestamp", e); + } + mathString = text.substring("now".length()); + } else { + int index = text.indexOf("||"); + if (index == -1) { + // no math, just parse date + // Normalize input for case-insensitive ISO datetime handling + text = normalizeDateMathInput(text); + return parseDateTime(text, timeZone, roundUpProperty); + } + time = parseDateTime(normalizeDateMathInput(text.substring(0, index).trim()), timeZone, false); + mathString = text.substring(index + 2).trim(); + } + + return parseMath(mathString, time, roundUpProperty, timeZone); + } + + private Instant parseMath(final String mathString, final Instant time, final boolean roundUpProperty, + ZoneId timeZone) throws DateMathParseException { + if (timeZone == null) { + timeZone = ZoneOffset.UTC; + } + ZonedDateTime dateTime = ZonedDateTime.ofInstant(time, timeZone); + int i = 0; + while (i < mathString.length()) { + char c = mathString.charAt(i++); + final boolean round; + final int sign; + if (c == '/') { + round = true; + sign = 1; + } else { + round = false; + if (c == '+') { + sign = 1; + } else if (c == '-') { + sign = -1; + } else { + throw new DateMathParseException("operator not supported for date math [%s]", mathString); + } + } + + if (i >= mathString.length()) { + throw new DateMathParseException("truncated date math [%s]", mathString); + } + + final int num; + int numStart = i; + if (!Character.isDigit(mathString.charAt(i))) { + num = 1; + } else { + while (i < mathString.length() && Character.isDigit(mathString.charAt(i))) { + i++; + } + if (i >= mathString.length()) { + throw new DateMathParseException("truncated date math [%s]", mathString); + } + num = Integer.parseInt(mathString.substring(numStart, i)); + } + if (round && num != 1) { + throw new DateMathParseException("rounding `/` can only be used on single unit types [%s]", mathString); + } + char unit = mathString.charAt(i++); + switch (unit) { + case 'y': + if (round) { + dateTime = dateTime.withDayOfYear(1).with(LocalTime.MIN); + if (roundUpProperty) { + dateTime = dateTime.plusYears(1); + } + } else { + dateTime = dateTime.plusYears(sign * num); + } + break; + case 'M': + if (round) { + dateTime = dateTime.withDayOfMonth(1).with(LocalTime.MIN); + if (roundUpProperty) { + dateTime = dateTime.plusMonths(1); + } + } else { + dateTime = dateTime.plusMonths(sign * num); + } + break; + case 'w': + if (round) { + dateTime = dateTime.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).with(LocalTime.MIN); + if (roundUpProperty) { + dateTime = dateTime.plusWeeks(1); + } + } else { + dateTime = dateTime.plusWeeks(sign * num); + } + break; + case 'd': + if (round) { + dateTime = dateTime.with(LocalTime.MIN); + if (roundUpProperty) { + dateTime = dateTime.plusDays(1); + } + } else { + dateTime = dateTime.plusDays(sign * num); + } + break; + case 'h': + case 'H': + if (round) { + dateTime = dateTime.withMinute(0).withSecond(0).withNano(0); + if (roundUpProperty) { + dateTime = dateTime.plusHours(1); + } + } else { + dateTime = dateTime.plusHours(sign * num); + } + break; + case 'm': + if (round) { + dateTime = dateTime.withSecond(0).withNano(0); + if (roundUpProperty) { + dateTime = dateTime.plusMinutes(1); + } + } else { + dateTime = dateTime.plusMinutes(sign * num); + } + break; + case 's': + if (round) { + dateTime = dateTime.withNano(0); + if (roundUpProperty) { + dateTime = dateTime.plusSeconds(1); + } + } else { + dateTime = dateTime.plusSeconds(sign * num); + } + break; + default: + // Adjust error message to remove the operator sign from the substring + // We know substring from numStart to current i is the "1X" part + // Operator was c, num was parsed, unit is unit + String unitString = mathString.substring(numStart, numStart + Integer.toString(num).length()) + unit; + throw new DateMathParseException("unit [%s] not supported for date math [%s]", unit, unitString); + } + if (round && roundUpProperty) { + // subtract 1 millisecond + dateTime = dateTime.minus(1, ChronoField.MILLI_OF_SECOND.getBaseUnit()); + } + } + return dateTime.toInstant(); + } + + private Instant parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNoTime) { + if (isNullOrEmpty(value)) { + throw new DateMathParseException("cannot parse empty date"); + } + + Function parser = roundUpIfNoTime ? roundUpFormatter::parse : formatter::parse; + try { + TemporalAccessor accessor = parser.apply(value); + + // Convert to ZonedDateTime from accessor + ZonedDateTime zdt = DateFormatters.from(accessor); + if (timeZone != null) { + // Convert to the same instant in the given timeZone + zdt = zdt.withZoneSameInstant(timeZone); + } + return zdt.toInstant(); + } catch (Throwable t) { + throw new DateMathParseException( + "failed to parse date field [%s] with format [%s]: [%s]", + value, formatter, t.getMessage()); + } + } + +} diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/JavaDateFormatter.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/JavaDateFormatter.java new file mode 100644 index 0000000000..bc21e39086 --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/datemath/JavaDateFormatter.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions.datemath; + +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeFormatterBuilder; +import java.time.temporal.ChronoField; +import java.time.temporal.TemporalAccessor; +import java.time.temporal.TemporalQueries; +import java.util.ArrayList; +import java.util.List; + +public class JavaDateFormatter { + private final List formats; + private final boolean allowEpochMillis; + private final boolean allowEpochSecond; + + public JavaDateFormatter(String formatString) { + this.formats = new ArrayList<>(); + boolean epochMillis = false; + boolean epochSecond = false; + + String[] formatParts = formatString.split("\\|\\|"); + for (String f : formatParts) { + f = f.trim(); + if (f.equals("epoch_millis")) { + formats.add(new FormatDefinition("epoch_millis", f, null)); + epochMillis = true; + } else if (f.equals("epoch_second")) { + formats.add(new FormatDefinition("epoch_second", f, null)); + epochSecond = true; + } else { + formats.add(createFormatDefinition(f)); + } + } + + this.allowEpochMillis = epochMillis; + this.allowEpochSecond = epochSecond; + } + + private String adjustForCaseInsensitive(String input) { + // Replace 't' with 'T' only when it's part of an ISO datetime format (e.g., `2022-05-18t15:23:17z`) + input = input.replaceAll("(?<=\\d{4}-\\d{2}-\\d{2})t", "T"); // Match 't' after a full date + // Replace 'z' with 'Z' only when it's at the end of the string or follows time components + input = input.replaceAll("z$", "Z"); // Match 'z' at the end + input = input.replaceAll("(?<=[:\\d])z", "Z"); // Match 'z' after a time component + return input; + } + + public TemporalAccessor parse(String input) { + // Numeric check + if (isNumeric(input)) { + long value = Long.parseLong(input); + if (allowEpochMillis && containsFormatName("epoch_millis")) { + return Instant.ofEpochMilli(value); + } + if (allowEpochSecond && containsFormatName("epoch_second")) { + return Instant.ofEpochSecond(value); + } + } + + input = adjustForCaseInsensitive(input); + + for (FormatDefinition def : formats) { + try { + String adjusted = adjustForPattern(input, def.pattern); + TemporalAccessor ta = def.formatter.parse(adjusted); + return toInstant(ta, def.formatter.getZone() != null ? def.formatter.getZone() : ZoneOffset.UTC); + } catch (Throwable t) { + // try next + } + } + + throw new DateMathParseException("failed to parse date field [" + input + "] with provided formats"); + } + + private Instant toInstant(TemporalAccessor ta, ZoneId zone) { + boolean hasYear = ta.isSupported(ChronoField.YEAR); + boolean hasMonth = ta.isSupported(ChronoField.MONTH_OF_YEAR); + boolean hasDay = ta.isSupported(ChronoField.DAY_OF_MONTH); + boolean hasDoY = ta.isSupported(ChronoField.DAY_OF_YEAR); + boolean hasHour = ta.isSupported(ChronoField.HOUR_OF_DAY); + boolean hasMinute = ta.isSupported(ChronoField.MINUTE_OF_HOUR); + boolean hasSecond = ta.isSupported(ChronoField.SECOND_OF_MINUTE); + + LocalDate date; + if (hasYear && hasMonth && hasDay) { + // Normal date + date = LocalDate.from(ta); + } else if (hasYear && hasDoY) { + // Ordinal date: year + dayOfYear + int year = ta.get(ChronoField.YEAR); + int dayOfYear = ta.get(ChronoField.DAY_OF_YEAR); + date = LocalDate.ofYearDay(year, dayOfYear); + } else if (!hasYear && (hasHour || hasMinute || hasSecond)) { + // Time only → 1970-01-01 + date = LocalDate.ofEpochDay(0); + } else if (hasYear && !hasMonth && !hasDay && !hasDoY) { + // Year only → yyyy means yyyy-01-01 + int year = ta.get(ChronoField.YEAR); + date = LocalDate.of(year, 1, 1); + } else { + // Maybe week-based fields or partial fields: + // Attempt ZonedDateTime directly if possible + try { + return ZonedDateTime.from(ta).toInstant(); + } catch (DateTimeException e) { + // If fails, handle week-based or incomplete dates + if (ta.isSupported(ChronoField.YEAR_OF_ERA)) { + throw new DateMathParseException("Week-based date formats need additional logic."); + } + // Default to epoch day + date = LocalDate.ofEpochDay(0); + } + } + + int hour = hasHour ? ta.get(ChronoField.HOUR_OF_DAY) : 0; + int minute = hasMinute ? ta.get(ChronoField.MINUTE_OF_HOUR) : 0; + int second = hasSecond ? ta.get(ChronoField.SECOND_OF_MINUTE) : 0; + int nano = ta.isSupported(ChronoField.NANO_OF_SECOND) ? ta.get(ChronoField.NANO_OF_SECOND) : 0; + + LocalTime time = LocalTime.of(hour, minute, second, nano); + + // Handle zone and offset explicitly + ZoneOffset offset = ta.query(TemporalQueries.offset()); + if (offset != null) { + return ZonedDateTime.of(date, time, offset).toInstant(); + } + + // Fall back to provided zone + return ZonedDateTime.of(date, time, zone).toInstant(); + } + + private String adjustForPattern(String input, String pattern) { + // If pattern is strict_date_* and only date is given, append midnight + if (pattern.contains("strict_date") && input.matches("^\\d{4}-\\d{2}-\\d{2}$")) { + return input + "T00:00:00Z"; + } + return input; + } + + private boolean isNumeric(String s) { + if (s.isEmpty()) return false; + for (char c : s.toCharArray()) { + if (!Character.isDigit(c)) return false; + } + return true; + } + + private boolean containsFormatName(String name) { + return formats.stream().anyMatch(def -> def.name.equals(name)); + } + + private FormatDefinition createFormatDefinition(String f) { + // Known patterns from documentation: + // We'll define exact patterns for each built-in format: + switch (f) { + // Already handled: epoch_millis, epoch_second + case "strict_date_optional_time": + case "date_optional_time": + // yyyy-MM-dd or yyyy-MM-dd'T'HH:mm:ss.SSSX + return fmt(f, "yyyy-MM-dd['T'HH:mm:ss[.SSS][XXX]]"); + + case "strict_date_optional_time_nanos": + // Nanosecond resolution: allow up to 9 fractional digits: .SSSSSSSSS + return fmt(f, "yyyy-MM-dd['T'HH:mm:ss[.SSSSSSSSS]][X]"); + + case "basic_date": + return fmt(f, "yyyyMMdd"); + case "basic_date_time": + return fmt(f, "yyyyMMdd'T'HHmmss.SSSX"); + case "basic_date_time_no_millis": + return fmt(f, "yyyyMMdd'T'HHmmssX"); + case "basic_ordinal_date": + return fmt(f, "yyyyDDD"); + case "basic_ordinal_date_time": + return fmt(f, "yyyyDDD'T'HHmmss.SSSX"); + case "basic_ordinal_date_time_no_millis": + return fmt(f, "yyyyDDD'T'HHmmssX"); + case "basic_time": + return fmt(f, "HHmmss.SSSX"); + case "basic_time_no_millis": + return fmt(f, "HHmmssX"); + case "basic_t_time": + return fmt(f, "'T'HHmmss.SSSX"); + case "basic_t_time_no_millis": + return fmt(f, "'T'HHmmssX"); + + // Week-based formats: + // Week dates require 'YYYY' for weekyear, 'ww' for week of year, and 'e' for day of week. + // Example: basic_week_date: xxxx'W'wwe + // We'll assume ISO week date parsing works with pattern: + case "basic_week_date": + case "strict_basic_week_date": + return fmt(f, "xxxx'W'wwe"); + case "basic_week_date_time": + case "strict_basic_week_date_time": + return fmt(f, "xxxx'W'wwe'T'HHmmss.SSSX"); + case "basic_week_date_time_no_millis": + case "strict_basic_week_date_time_no_millis": + return fmt(f, "xxxx'W'wwe'T'HHmmssX"); + + case "date": + case "strict_date": + return fmt(f, "yyyy-MM-dd"); + case "date_hour": + case "strict_date_hour": + return fmt(f, "yyyy-MM-dd'T'HH"); + case "date_hour_minute": + case "strict_date_hour_minute": + return fmt(f, "yyyy-MM-dd'T'HH:mm"); + case "date_hour_minute_second": + case "strict_date_hour_minute_second": + return fmt(f, "yyyy-MM-dd'T'HH:mm:ss"); + case "date_hour_minute_second_fraction": + case "strict_date_hour_minute_second_fraction": + return fmt(f, "yyyy-MM-dd'T'HH:mm:ss.SSS"); + case "date_hour_minute_second_millis": + case "strict_date_hour_minute_second_millis": + // same as fraction? + return fmt(f, "yyyy-MM-dd'T'HH:mm:ss.SSS"); + case "date_time": + case "strict_date_time": + return fmt(f, "yyyy-MM-dd'T'HH:mm:ss.SSSX"); + case "date_time_no_millis": + case "strict_date_time_no_millis": + return fmt(f, "yyyy-MM-dd'T'HH:mm:ssXXX"); + + case "hour": + case "strict_hour": + return fmt(f, "HH"); + case "hour_minute": + case "strict_hour_minute": + return fmt(f, "HH:mm"); + case "hour_minute_second": + case "strict_hour_minute_second": + return fmt(f, "HH:mm:ss"); + case "hour_minute_second_fraction": + case "strict_hour_minute_second_fraction": + return fmt(f, "HH:mm:ss.SSS"); + case "hour_minute_second_millis": + case "strict_hour_minute_second_millis": + return fmt(f, "HH:mm:ss.SSS"); + case "ordinal_date": + case "strict_ordinal_date": + return fmt(f, "yyyy-DDD"); + case "ordinal_date_time": + case "strict_ordinal_date_time": + return fmt(f, "yyyy-DDD'T'HH:mm:ss.SSSX"); + case "ordinal_date_time_no_millis": + case "strict_ordinal_date_time_no_millis": + return fmt(f, "yyyy-DDD'T'HH:mm:ssX"); + case "time": + case "strict_time": + return fmt(f, "HH:mm:ss.SSSX"); + case "time_no_millis": + case "strict_time_no_millis": + return fmt(f, "HH:mm:ssX"); + case "t_time": + case "strict_t_time": + return fmt(f, "'T'HH:mm:ss.SSSX"); + case "t_time_no_millis": + case "strict_t_time_no_millis": + return fmt(f, "'T'HH:mm:ssX"); + case "week_date": + case "strict_week_date": + return fmt(f, "YYYY-'W'ww-e"); + case "week_date_time": + case "strict_week_date_time": + return fmt(f, "YYYY-'W'ww-e'T'HH:mm:ss.SSSX"); + case "week_date_time_no_millis": + case "strict_week_date_time_no_millis": + return fmt(f, "YYYY-'W'ww-e'T'HH:mm:ssX"); + case "weekyear": + case "strict_weekyear": + return fmt(f, "YYYY"); + case "weekyear_week": + case "strict_weekyear_week": + return fmt(f, "YYYY-'W'ww"); + case "weekyear_week_day": + case "strict_weekyear_week_day": + return fmt(f, "YYYY-'W'ww-e"); + case "year": + case "strict_year": + return fmt(f, "yyyy"); + case "year_month": + case "strict_year_month": + return fmt(f, "yyyy-MM"); + case "year_month_day": + case "strict_year_month_day": + return fmt(f, "yyyy-MM-dd"); + + default: + // Custom pattern + return fmt(f, f); + } + } + + private FormatDefinition fmt(String name, String pattern) { + // Apply UTC zone to all and consider using strict resolver if needed + DateTimeFormatter dtf = new DateTimeFormatterBuilder() + .parseCaseSensitive() + .appendPattern(pattern) + .toFormatter() + .withZone(ZoneOffset.UTC); + return new FormatDefinition(name, pattern, dtf); + } + + public static class FormatDefinition { + final String name; + final String pattern; + final DateTimeFormatter formatter; + + public FormatDefinition(String name, String pattern, DateTimeFormatter formatter) { + this.name = name; + this.pattern = pattern; + this.formatter = formatter; + } + + @Override + public String toString() { + return "FormatDefinition{" + + "name='" + name + '\'' + + ", pattern='" + pattern + '\'' + + '}'; + } + } + + + @Override + public String toString() { + return "JavaDateFormatter{" + + "formats=" + formats + + '}'; + } +} diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/geo/DistanceUnit.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/geo/DistanceUnit.java new file mode 100644 index 0000000000..292cbd383f --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/geo/DistanceUnit.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions.geo; + +import java.util.HashMap; +import java.util.Map; + +public enum DistanceUnit { + KILOMETERS(1000.0, "km", "kilometers"), + MILES(1609.344, "mi", "miles"), + YARDS(0.9144, "yd", "yards"), + FEET(0.3048, "ft", "feet"), + INCHES(0.0254, "in", "inches"), + NAUTICAL_MILES(1852.0, "NM", "nauticalmiles"), + METERS(1.0, "m", "meters"); + + // Constants for Earth's properties + private static final double EARTH_SEMI_MAJOR_AXIS = 6378137.0; // in meters + private static final double EARTH_EQUATOR = 2*Math.PI * EARTH_SEMI_MAJOR_AXIS; // in meters + + private static final Map UNIT_MAP = new HashMap<>(); + + public static final DistanceUnit DEFAULT = METERS; + + static { + for (DistanceUnit unit : values()) { + for (String alias : unit.aliases) { + UNIT_MAP.put(alias.toLowerCase(), unit); + } + } + } + + private final double metersPerUnit; + private final String[] aliases; + + DistanceUnit(double metersPerUnit, String... aliases) { + this.metersPerUnit = metersPerUnit; + this.aliases = aliases; + } + + public double getEarthCircumference() { + return EARTH_EQUATOR / metersPerUnit; + } + + public double getEarthRadius() { + return EARTH_SEMI_MAJOR_AXIS / metersPerUnit; + } + + public double getDistancePerDegree() { + return EARTH_EQUATOR / (360.0 * metersPerUnit); + } + + public double toMeters(double value) { + return value * metersPerUnit; + } + + public double fromMeters(double value) { + return value / metersPerUnit; + } + + public double convert(double value, DistanceUnit toUnit) { + return (value * metersPerUnit) / toUnit.metersPerUnit; + } + + public static double convert(double value, DistanceUnit from, DistanceUnit to) { + return (value * from.metersPerUnit) / to.metersPerUnit; + } + + public static DistanceUnit fromString(String unit) { + if (unit == null || unit.isEmpty()) { + throw new IllegalArgumentException("Unit string must not be null or empty"); + } + DistanceUnit distanceUnit = UNIT_MAP.get(unit.toLowerCase()); + if (distanceUnit == null) { + throw new IllegalArgumentException("Unknown distance unit: " + unit); + } + return distanceUnit; + } + + public static DistanceUnit parseUnit(String distance, DistanceUnit defaultUnit) { + for (DistanceUnit unit : values()) { + for (String alias : unit.aliases) { + if (distance.endsWith(alias)) { + return unit; + } + } + } + return defaultUnit; + } + + public double parse(String distance, DistanceUnit defaultUnit) { + Distance parsed = Distance.parseDistance(distance, defaultUnit); + return convert(parsed.value, parsed.unit, this); + } + + @Override + public String toString() { + return aliases[0]; + } + + public static class Distance { + public final double value; + public final DistanceUnit unit; + + public Distance(double value, DistanceUnit unit) { + this.value = value; + this.unit = unit; + } + + public Distance convert(DistanceUnit toUnit) { + double convertedValue = DistanceUnit.convert(value, unit, toUnit); + return new Distance(convertedValue, toUnit); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + Distance other = (Distance) obj; + return Double.compare(DistanceUnit.convert(value, unit, other.unit), other.value) == 0; + } + + @Override + public int hashCode() { + return Double.hashCode(value * unit.metersPerUnit); + } + + @Override + public String toString() { + return value + " " + unit.toString(); + } + + public static Distance parseDistance(String distance) { + return parseDistance(distance, DistanceUnit.METERS); + } + + public static Distance parseDistance(String distance, DistanceUnit defaultUnit) { + for (DistanceUnit unit : values()) { + for (String alias : unit.aliases) { + if (distance.endsWith(alias)) { + String valuePart = distance.substring(0, distance.length() - alias.length()).trim(); + return new Distance(Double.parseDouble(valuePart), unit); + } + } + } + return new Distance(Double.parseDouble(distance), defaultUnit); + } + } +} diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/geo/GeoDistance.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/geo/GeoDistance.java new file mode 100644 index 0000000000..117671c022 --- /dev/null +++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/conditions/geo/GeoDistance.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.unomi.persistence.spi.conditions.geo; + +public enum GeoDistance { + HAVERSINE { + @Override + public double calculate(double lat1, double lon1, double lat2, double lon2, DistanceUnit unit) { + double latRad1 = toRadians(lat1); + double latRad2 = toRadians(lat2); + double deltaLat = toRadians(lat2 - lat1); + double deltaLon = toRadians(lon2 - lon1); + + double a = Math.sin(deltaLat / 2) * Math.sin(deltaLat / 2) + + Math.cos(latRad1) * Math.cos(latRad2) + * Math.sin(deltaLon / 2) * Math.sin(deltaLon / 2); + + double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a)); + return unit.convert(EARTH_MEAN_RADIUS * c, DistanceUnit.METERS); + } + }, + ARC { + @Override + public double calculate(double lat1, double lon1, double lat2, double lon2, DistanceUnit unit) { + double latRad1 = toRadians(lat1); + double lonRad1 = toRadians(lon1); + double latRad2 = toRadians(lat2); + double lonRad2 = toRadians(lon2); + + double deltaLon = lonRad2 - lonRad1; + + double cosTheta = Math.sin(latRad1) * Math.sin(latRad2) + + Math.cos(latRad1) * Math.cos(latRad2) * Math.cos(deltaLon); + + double theta = Math.acos(Math.min(1.0, Math.max(-1.0, cosTheta))); // Clamp to avoid NaN + return unit.convert(EARTH_MEAN_RADIUS * theta, DistanceUnit.METERS); + } + }, + PLANE { + @Override + public double calculate(double lat1, double lon1, double lat2, double lon2, DistanceUnit unit) { + double x = toRadians(lon2 - lon1) * Math.cos(toRadians((lat1 + lat2) / 2)); + double y = toRadians(lat2 - lat1); + double distance = Math.sqrt(x * x + y * y) * EARTH_MEAN_RADIUS; + return unit.convert(distance, DistanceUnit.METERS); + } + }; + + public static final double EARTH_MEAN_RADIUS = 6371008.7714D; // meters (WGS 84) + + public static final double TO_RADIANS = Math.PI / 180D; + public static final double TO_DEGREES = 180D / Math.PI; + + public static double toRadians(double degrees) { + return degrees * TO_RADIANS; + } + + /** + * Calculate the distance between two geographic points and return it in the specified unit. + * + * @param lat1 Latitude of the first point. + * @param lon1 Longitude of the first point. + * @param lat2 Latitude of the second point. + * @param lon2 Longitude of the second point. + * @param unit The distance unit for the result. + * @return The calculated distance in the specified unit. + */ + public abstract double calculate(double lat1, double lon1, double lat2, double lon2, DistanceUnit unit); + + /** + * Converts a string representation of a GeoDistance method to its corresponding enum. + * + * @param name The string representation of the GeoDistance method (e.g., "plane" or "arc"). + * @return The corresponding GeoDistance enum. + * @throws IllegalArgumentException If the name does not match any known GeoDistance methods. + */ + public static GeoDistance fromString(String name) { + if ("plane".equalsIgnoreCase(name)) { + return PLANE; + } else if ("arc".equalsIgnoreCase(name)) { + return ARC; + } else if ("haversine".equalsIgnoreCase(name)) { + return HAVERSINE; + } else { + throw new IllegalArgumentException("Unknown GeoDistance method: " + name); + } + } +} diff --git a/persistence-spi/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/persistence-spi/src/main/resources/OSGI-INF/blueprint/blueprint.xml new file mode 100644 index 0000000000..7da5a271f8 --- /dev/null +++ b/persistence-spi/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher + + + + \ No newline at end of file diff --git a/persistence-elasticsearch/core/src/main/resources/mapping-FoldToASCII.txt b/persistence-spi/src/main/resources/mapping-FoldToASCII.txt similarity index 99% rename from persistence-elasticsearch/core/src/main/resources/mapping-FoldToASCII.txt rename to persistence-spi/src/main/resources/mapping-FoldToASCII.txt index 9a84b6eac3..c9c69dd0bd 100644 --- a/persistence-elasticsearch/core/src/main/resources/mapping-FoldToASCII.txt +++ b/persistence-spi/src/main/resources/mapping-FoldToASCII.txt @@ -3810,4 +3810,4 @@ # @source_char_descriptions = (); # $target = ''; # } -# } +# } \ No newline at end of file diff --git a/persistence-spi/src/test/java/conditions/datemath/DateMathParserTest.java b/persistence-spi/src/test/java/conditions/datemath/DateMathParserTest.java new file mode 100644 index 0000000000..d60bc8b922 --- /dev/null +++ b/persistence-spi/src/test/java/conditions/datemath/DateMathParserTest.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package conditions.datemath; + +import org.apache.unomi.persistence.spi.conditions.datemath.DateMathParseException; +import org.apache.unomi.persistence.spi.conditions.datemath.DateMathParser; +import org.apache.unomi.persistence.spi.conditions.datemath.JavaDateFormatter; +import org.junit.Test; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.function.LongSupplier; + +import static org.junit.Assert.*; + +public class DateMathParserTest { + + // Create the JavaDateFormatter with epoch millis support + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_optional_time||epoch_millis"); + + // Round up formatter can be the same or similar: + DateTimeFormatter roundUpFormatter = DateTimeFormatter.ISO_DATE_TIME; + + DateMathParser parser = new DateMathParser(formatter, roundUpFormatter); + + // A fixed "now" supplier returning a fixed timestamp: 2001-01-01T12:00:00Z in epoch millis + private final LongSupplier fixedNow = () -> ZonedDateTime.of(2001, 1, 1, 12, 0, 0, 0, ZoneOffset.UTC).toInstant().toEpochMilli(); + + @Test + public void testNowWithFixedSupplier() { + Instant parsed = parser.parse("now", fixedNow, false, ZoneOffset.UTC); + assertEquals("2001-01-01T12:00:00Z", parsed.toString()); + } + + @Test + public void testNowPlus1HourWithFixedSupplier() { + Instant parsed = parser.parse("now+1h", fixedNow, false, ZoneOffset.UTC); + assertEquals("2001-01-01T13:00:00Z", parsed.toString()); + } + + @Test + public void testNowMinus1HourWithFixedSupplier() { + Instant parsed = parser.parse("now-1h", fixedNow, false, ZoneOffset.UTC); + assertEquals("2001-01-01T11:00:00Z", parsed.toString()); + } + + @Test + public void testRoundingWithRoundUpProperty() { + // now = 2001-01-01T12:00:00Z + // now/d with roundUp = true should round to the beginning of the day + 1 day -1 millisecond + // Actually: rounding sets time to 2001-01-01T00:00:00Z, roundUp would move to 2001-01-02T00:00:00Z, + // and subtract one millisecond → 2001-01-01T23:59:59.999Z + Instant parsed = parser.parse("now/d", fixedNow, true, ZoneOffset.UTC); + // Just before midnight of the next day + assertEquals("2001-01-01T23:59:59.999Z", parsed.toString()); + } + + @Test + public void testRoundingWithoutRoundUpProperty() { + Instant parsed = parser.parse("now/d", fixedNow, false, ZoneOffset.UTC); + // Rounding down to the start of the day: 2001-01-01T00:00:00Z + assertEquals("2001-01-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testFixedDatePlusMonthAndRoundToDay() { + // "2001-02-01||+1M/d" + // start: 2001-02-01T00:00:00Z + // +1M → 2001-03-01T00:00:00Z + // /d rounds to start of day (same day) + Instant parsed = parser.parse("2001-02-01||+1M/d", fixedNow, false, ZoneOffset.UTC); + assertEquals("2001-03-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testInvalidUnit() { + try { + parser.parse("now+1X", fixedNow, false, ZoneOffset.UTC); + fail("Expected an exception"); + } catch (DateMathParseException e) { + // "Invalid unit: X" + // Actually from code: "unit [X] not supported for date math [1X]" + // The exact message: + // "unit [X] not supported for date math [1X]" is expected + // The code tries to parse "now+1X" → operator '+' recognized, num=1, unit='X'. + // The mathString is "1X". We'll accept the default message. + assertEquals("unit [X] not supported for date math [1X]", e.getMessage()); + } + } + + @Test + public void testInvalidOperator() { + try { + parser.parse("now*1d", fixedNow, false, ZoneOffset.UTC); + fail("Expected an exception"); + } catch (DateMathParseException e) { + // operator not supported + assertEquals("operator not supported for date math [*1d]", e.getMessage()); + } + } + + @Test + public void testTruncatedMath() { + try { + parser.parse("now+", fixedNow, false, ZoneOffset.UTC); + fail("Expected an exception"); + } catch (DateMathParseException e) { + // truncated date math + assertEquals("truncated date math [+]", e.getMessage()); + } + } + + @Test + public void testRoundSingleUnitOnly() { + try { + parser.parse("now/2d", fixedNow, false, ZoneOffset.UTC); + fail("Expected an exception"); + } catch (DateMathParseException e) { + // rounding `/` can only be used on single unit types + assertEquals("rounding `/` can only be used on single unit types [/2d]", e.getMessage()); + } + } + + @Test + public void testTimeZoneAwareParsing() { + // Parse a date with a different timezone + // We'll pick a date in a different zone and ensure it adjusts if timeZone is provided + Instant parsed = parser.parse("2001-01-01T12:00:00-02:00", fixedNow, false, ZoneOffset.UTC); + // 12:00 at -02:00 is actually 14:00 UTC + assertEquals("2001-01-01T14:00:00Z", parsed.toString()); + } + + @Test + public void testNowWithMultipleOperationsAndRoundUp() { + // now = 2001-01-01T12:00:00Z + // now+1M-1d/d with roundUp = true + // Steps: + // +1M → 2001-02-01T12:00:00Z + // -1d → 2001-01-31T12:00:00Z + // /d with roundUp → round down: 2001-01-31T00:00:00Z plus 1 day = 2001-02-01T00:00:00Z minus 1 ms = 2001-01-31T23:59:59.999Z + Instant parsed = parser.parse("now+1M-1d/d", fixedNow, true, ZoneOffset.UTC); + assertEquals("2001-01-31T23:59:59.999Z", parsed.toString()); + } + + @Test + public void testEmptyDate() { + try { + parser.parse("", fixedNow, false, ZoneOffset.UTC); + fail("Expected an exception"); + } catch (DateMathParseException e) { + assertEquals("cannot parse empty date", e.getMessage()); + } + } + + @Test + public void testInvalidExpression() { + // Something like 2022-05-18||invalid + try { + parser.parse("2022-05-18||invalid", fixedNow, false, ZoneOffset.UTC); + fail("Expected an exception"); + } catch (DateMathParseException e) { + // operator not supported or truncated math. Actually, 'i' is not recognized as '+', '-', or '/' + // This should fail as "operator not supported for date math [invalid]" + assertEquals("operator not supported for date math [invalid]", e.getMessage()); + } + } + + @Test + public void testFailedToParseDate() { + try { + parser.parse("not-a-date", fixedNow, false, ZoneOffset.UTC); + fail("Expected an exception"); + } catch (DateMathParseException e) { + // "failed to parse date field [not-a-date] with format [strict_date_optional_time]: [Text 'not-a-date' could not be parsed...]" + // Checking just the start of the message: + assertTrue(e.getMessage().startsWith("failed to parse date field [not-a-date] with format")); + } + } + + @Test + public void testInvalidLowercaseMathOperator() { + try { + parser.parse("now*1d", fixedNow, false, ZoneOffset.UTC); // Invalid operator + fail("Expected an exception"); + } catch (DateMathParseException e) { + assertEquals("operator not supported for date math [*1d]", e.getMessage()); + } + } + + + @Test + public void testDateMathWithCaseInsensitiveParsing() { + Instant parsed = parser.parse("2001-01-01t12:00:00z||+1d", fixedNow, false, ZoneOffset.UTC); + assertEquals("2001-01-02T12:00:00Z", parsed.toString()); + + parsed = parser.parse("now+1h/d", fixedNow, false, ZoneOffset.UTC); + assertEquals("2001-01-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testMixedCaseDateMath() { + Instant parsed = parser.parse("2001-01-01T12:00:00z||+1M/d", fixedNow, true, ZoneOffset.UTC); // Mixed case + assertEquals("2001-02-01T23:59:59.999Z", parsed.toString()); + } + + @Test + public void testInvalidMathWithCaseInsensitiveInput() { + try { + parser.parse("now*1d", fixedNow, false, ZoneOffset.UTC); // Invalid operator + fail("Expected an exception"); + } catch (DateMathParseException e) { + assertEquals("operator not supported for date math [*1d]", e.getMessage()); + } + + try { + parser.parse("2001-01-01t12:00:00x||+1d", fixedNow, false, ZoneOffset.UTC); // Invalid separator + fail("Expected an exception"); + } catch (DateMathParseException e) { + assertTrue(e.getMessage().contains("failed to parse date field")); + } + } + + @Test + public void testDateMathWithExtraSpaces() { + Instant parsed = parser.parse(" 2001-01-01T12:00:00Z || +1d ", fixedNow, false, ZoneOffset.UTC); + assertEquals("2001-01-02T12:00:00Z", parsed.toString()); + } + +} diff --git a/persistence-spi/src/test/java/conditions/datemath/JavaDateFormatterTest.java b/persistence-spi/src/test/java/conditions/datemath/JavaDateFormatterTest.java new file mode 100644 index 0000000000..655a20b4ef --- /dev/null +++ b/persistence-spi/src/test/java/conditions/datemath/JavaDateFormatterTest.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package conditions.datemath; + +import org.apache.unomi.persistence.spi.conditions.datemath.DateMathParseException; +import org.apache.unomi.persistence.spi.conditions.datemath.JavaDateFormatter; +import org.junit.Test; + +import java.time.Instant; + +import static org.junit.Assert.*; + +/** + * Comprehensive tests for JavaDateFormatter covering various formats: + * - Epoch formats (epoch_millis, epoch_second) + * - ISO-based formats (strict_date_optional_time, strict_date_time_no_millis, etc.) + * - Basic formats (basic_date, basic_date_time, etc.) + * - Ordinal formats (ordinal_date, etc.) + * - Strict vs non-strict variants + * - Custom patterns + * - Fallback between multiple formats + */ +public class JavaDateFormatterTest { + + @Test + public void testEpochMillis() { + // epoch_millis: 978307200000L = 2001-01-01T12:00:00Z + // The expected original code had midnight, let's confirm correct epoch: + // 978307200000 ms = 2001-01-01T12:00:00Z indeed. + JavaDateFormatter formatter = new JavaDateFormatter("epoch_millis"); + Instant parsed = Instant.from(formatter.parse("978307200000")); + assertEquals("2001-01-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testEpochSecond() { + // epoch_second: 978307200 = 2001-01-01T12:00:00Z + JavaDateFormatter formatter = new JavaDateFormatter("epoch_second"); + Instant parsed = Instant.from(formatter.parse("978307200")); + assertEquals("2001-01-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testStrictDateOptionalTime() { + // strict_date_optional_time: "2022-05-18T15:23:17Z" + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_optional_time"); + Instant parsed = Instant.from(formatter.parse("2022-05-18T15:23:17Z")); + assertEquals("2022-05-18T15:23:17Z", parsed.toString()); + } + + @Test + public void testDateOnlyWithStrictDateOptionalTime() { + // strict_date_optional_time: "2022-05-18" should default to midnight + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_optional_time"); + Instant parsed = Instant.from(formatter.parse("2022-05-18")); + assertEquals("2022-05-18T00:00:00Z", parsed.toString()); + } + + @Test + public void testDateTimeNoMillis() { + // date_time_no_millis: "yyyy-MM-dd'T'HH:mm:ssZ" + JavaDateFormatter formatter = new JavaDateFormatter("date_time_no_millis"); + Instant parsed = Instant.from(formatter.parse("2022-05-18T15:23:17+00:00")); + assertEquals("2022-05-18T15:23:17Z", parsed.toString()); + } + + @Test + public void testStrictDateTimeNoMillis() { + // strict_date_time_no_millis: same pattern but strict + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_time_no_millis"); + Instant parsed = Instant.from(formatter.parse("2022-05-18T15:23:17Z")); + assertEquals("2022-05-18T15:23:17Z", parsed.toString()); + } + + @Test + public void testBasicDate() { + // basic_date: yyyyMMdd + // "20010101" = 2001-01-01T00:00:00Z + JavaDateFormatter formatter = new JavaDateFormatter("basic_date"); + Instant parsed = Instant.from(formatter.parse("20010101")); + assertEquals("2001-01-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testBasicDateTime() { + // basic_date_time: yyyyMMdd'T'HHmmss.SSSZ + // "20010101T123000.000Z" = 2001-01-01T12:30:00Z + JavaDateFormatter formatter = new JavaDateFormatter("basic_date_time"); + Instant parsed = Instant.from(formatter.parse("20010101T123000.000Z")); + assertEquals("2001-01-01T12:30:00Z", parsed.toString()); + } + + @Test + public void testOrdinalDate() { + // ordinal_date: yyyy-DDD, e.g. "2001-001" = 2001-01-01 + JavaDateFormatter formatter = new JavaDateFormatter("ordinal_date"); + Instant parsed = Instant.from(formatter.parse("2001-001")); + assertEquals("2001-01-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testOrdinalDateTimeNoMillis() { + // ordinal_date_time_no_millis: yyyy-DDD'T'HH:mm:ssZ + // "2001-001T12:00:00Z" = 2001-01-01T12:00:00Z + JavaDateFormatter formatter = new JavaDateFormatter("ordinal_date_time_no_millis"); + Instant parsed = Instant.from(formatter.parse("2001-001T12:00:00Z")); + assertEquals("2001-01-01T12:00:00Z", parsed.toString()); + } + + @Test + public void testHourMinuteSecond() { + // hour_minute_second: HH:mm:ss + // "12:34:56" with no date = defaults to today's date at that time? + // Our code sets date-only defaults. For time-only, must default to 1970-01-01? + // If not implemented, either skip or fix code to handle pure times. + // Let's assume we default to 1970-01-01 if time only: + JavaDateFormatter formatter = new JavaDateFormatter("hour_minute_second"); + Instant parsed = Instant.from(formatter.parse("12:34:56")); + assertEquals("1970-01-01T12:34:56Z", parsed.toString()); + } + + @Test + public void testCustomPattern() { + // "MM/dd/yyyy": "03/21/2019" = 2019-03-21T00:00:00Z + JavaDateFormatter formatter = new JavaDateFormatter("MM/dd/yyyy"); + Instant parsed = Instant.from(formatter.parse("03/21/2019")); + assertEquals("2019-03-21T00:00:00Z", parsed.toString()); + } + + @Test + public void testFallbackBetweenFormats() { + // If first format doesn't match, second one should + JavaDateFormatter formatter = new JavaDateFormatter("yyyy-MM-dd||epoch_millis"); + // Not a yyyy-MM-dd date, but epoch_millis: + Instant parsed = Instant.from(formatter.parse("978307200000")); + assertEquals("2001-01-01T00:00:00Z", parsed.toString()); + } + + @Test + public void testNoMatch() { + JavaDateFormatter formatter = new JavaDateFormatter("yyyy/MM/dd||basic_date"); + try { + formatter.parse("not-a-date"); + fail("Expected exception"); + } catch (DateMathParseException e) { + assertTrue(e.getMessage().contains("failed to parse date field [not-a-date]")); + } + } + + @Test + public void testMixedCaseDate() { + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_optional_time"); + Instant parsed = Instant.from(formatter.parse("2022-05-18T15:23:17z")); // mixed case 'T' and 'z' + assertEquals("2022-05-18T15:23:17Z", parsed.toString()); + } + + @Test + public void testCaseInsensitiveISOWithValidInputs() { + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_optional_time"); + // Lowercase `t` and `z` + Instant parsed = Instant.from(formatter.parse("2022-05-18t15:23:17z")); + assertEquals("2022-05-18T15:23:17Z", parsed.toString()); + + // Mixed case + parsed = Instant.from(formatter.parse("2022-05-18T15:23:17z")); + assertEquals("2022-05-18T15:23:17Z", parsed.toString()); + + // Uppercase (valid) + parsed = Instant.from(formatter.parse("2022-05-18T15:23:17Z")); + assertEquals("2022-05-18T15:23:17Z", parsed.toString()); + } + + @Test + public void testCaseInsensitiveISOWithInvalidInputs() { + JavaDateFormatter formatter = new JavaDateFormatter("strict_date_optional_time"); + + try { + formatter.parse("2022-05-18x15:23:17z"); // Invalid separator + fail("Expected an exception"); + } catch (DateMathParseException e) { + assertTrue(e.getMessage().contains("failed to parse date field")); + } + + try { + formatter.parse("2022-05-18T15:23:17X"); // Invalid character for timezone + fail("Expected an exception"); + } catch (DateMathParseException e) { + assertTrue(e.getMessage().contains("failed to parse date field")); + } + } +} diff --git a/persistence-spi/src/test/java/conditions/geo/DistanceUnitTest.java b/persistence-spi/src/test/java/conditions/geo/DistanceUnitTest.java new file mode 100644 index 0000000000..87be8cdb10 --- /dev/null +++ b/persistence-spi/src/test/java/conditions/geo/DistanceUnitTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package conditions.geo; + +import org.apache.unomi.persistence.spi.conditions.geo.DistanceUnit; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class DistanceUnitTest { + + // Local constants + /** Earth ellipsoid major axis defined by WGS 84 in meters */ + public static final double EARTH_SEMI_MAJOR_AXIS = 6378137.0; // meters (WGS 84) + + /** Earth ellipsoid minor axis defined by WGS 84 in meters */ + public static final double EARTH_SEMI_MINOR_AXIS = 6356752.314245; // meters (WGS 84) + + /** Earth mean radius defined by WGS 84 in meters */ + public static final double EARTH_MEAN_RADIUS = 6371008.7714D; // meters (WGS 84) + + /** Earth axis ratio defined by WGS 84 (0.996647189335) */ + public static final double EARTH_AXIS_RATIO = EARTH_SEMI_MINOR_AXIS / EARTH_SEMI_MAJOR_AXIS; + + /** Earth ellipsoid equator length in meters */ + public static final double EARTH_EQUATOR = 2*Math.PI * EARTH_SEMI_MAJOR_AXIS; + + /** Earth ellipsoid polar distance in meters */ + public static final double EARTH_POLAR_DISTANCE = Math.PI * EARTH_SEMI_MINOR_AXIS; + + @Test + public void testEarthCircumference() { + assertEquals(EARTH_EQUATOR / DistanceUnit.KILOMETERS.toMeters(1.0), DistanceUnit.KILOMETERS.getEarthCircumference(), 0.0001); + assertEquals(EARTH_EQUATOR, DistanceUnit.METERS.getEarthCircumference(), 0.0001); + } + + @Test + public void testEarthRadius() { + assertEquals(EARTH_SEMI_MAJOR_AXIS / DistanceUnit.MILES.toMeters(1.0), DistanceUnit.MILES.getEarthRadius(), 0.0001); + assertEquals(EARTH_SEMI_MAJOR_AXIS, DistanceUnit.METERS.getEarthRadius(), 0.0001); + } + + @Test + public void testDistancePerDegree() { + assertEquals(EARTH_EQUATOR / 360.0, DistanceUnit.METERS.getDistancePerDegree(), 0.0001); + assertEquals(EARTH_EQUATOR / (360.0 * DistanceUnit.KILOMETERS.toMeters(1.0)), DistanceUnit.KILOMETERS.getDistancePerDegree(), 0.0001); + } + + @Test + public void testToMeters() { + assertEquals(1000.0, DistanceUnit.KILOMETERS.toMeters(1.0), 0.0001); + assertEquals(1609.344, DistanceUnit.MILES.toMeters(1.0), 0.0001); + assertEquals(1.0, DistanceUnit.METERS.toMeters(1.0), 0.0001); + } + + @Test + public void testFromMeters() { + assertEquals(1.0, DistanceUnit.KILOMETERS.fromMeters(1000.0), 0.0001); + assertEquals(1.0, DistanceUnit.MILES.fromMeters(1609.344), 0.0001); + assertEquals(1.0, DistanceUnit.METERS.fromMeters(1.0), 0.0001); + } + + @Test + public void testConvert() { + assertEquals(1.0, DistanceUnit.convert(1.0, DistanceUnit.KILOMETERS, DistanceUnit.KILOMETERS), 0.0001); + assertEquals(1000.0, DistanceUnit.convert(1.0, DistanceUnit.KILOMETERS, DistanceUnit.METERS), 0.0001); + assertEquals(1.0, DistanceUnit.convert(1609.344, DistanceUnit.METERS, DistanceUnit.MILES), 0.0001); + } + + @Test + public void testToString() { + assertEquals("km", DistanceUnit.KILOMETERS.toString()); + assertEquals("mi", DistanceUnit.MILES.toString()); + } + + @Test + public void testParse() { + assertEquals(1.0, DistanceUnit.MILES.parse("1mi", DistanceUnit.METERS), 0.0001); + assertEquals(1609.344, DistanceUnit.METERS.parse("1mi", DistanceUnit.METERS), 0.0001); + assertEquals(1000.0, DistanceUnit.METERS.parse("1km", DistanceUnit.METERS), 0.0001); + assertEquals(1.0, DistanceUnit.KILOMETERS.parse("1km", DistanceUnit.KILOMETERS), 0.0001); + } + + @Test + public void testParseUnit() { + assertEquals(DistanceUnit.KILOMETERS, DistanceUnit.parseUnit("1km", DistanceUnit.METERS)); + assertEquals(DistanceUnit.MILES, DistanceUnit.parseUnit("1mi", DistanceUnit.METERS)); + assertEquals(DistanceUnit.METERS, DistanceUnit.parseUnit("1", DistanceUnit.METERS)); + } + + @Test + public void testDistanceParsing() { + DistanceUnit.Distance distance = DistanceUnit.Distance.parseDistance("1km"); + assertEquals(1.0, distance.value, 0.0001); + assertEquals(DistanceUnit.KILOMETERS, distance.unit); + + DistanceUnit.Distance defaultDistance = DistanceUnit.Distance.parseDistance("100"); + assertEquals(100.0, defaultDistance.value, 0.0001); + assertEquals(DistanceUnit.DEFAULT, defaultDistance.unit); + } + + @Test + public void testDistanceConversion() { + DistanceUnit.Distance distance = new DistanceUnit.Distance(1.0, DistanceUnit.KILOMETERS); + DistanceUnit.Distance converted = distance.convert(DistanceUnit.METERS); + assertEquals(1000.0, converted.value, 0.0001); + assertEquals(DistanceUnit.METERS, converted.unit); + } + + @Test + public void testDistanceEqualsAndHashCode() { + DistanceUnit.Distance d1 = new DistanceUnit.Distance(1.0, DistanceUnit.KILOMETERS); + DistanceUnit.Distance d2 = new DistanceUnit.Distance(1000.0, DistanceUnit.METERS); + assertEquals(d1, d2); + assertEquals(d1.hashCode(), d2.hashCode()); + } + + @Test(expected = IllegalArgumentException.class) + public void testFromStringInvalidUnit() { + DistanceUnit.fromString("invalid"); + } + +} diff --git a/persistence-spi/src/test/java/conditions/geo/GeoDistanceTest.java b/persistence-spi/src/test/java/conditions/geo/GeoDistanceTest.java new file mode 100644 index 0000000000..8b3d7e463f --- /dev/null +++ b/persistence-spi/src/test/java/conditions/geo/GeoDistanceTest.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package conditions.geo; + +import org.apache.unomi.persistence.spi.conditions.geo.DistanceUnit; +import org.apache.unomi.persistence.spi.conditions.geo.GeoDistance; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class GeoDistanceTest { + + private static final double SRC_LAT = 40.7128; // Example source latitude + private static final double SRC_LON = -74.0060; // Example source longitude + private static final double DST_LAT = 34.0522; // Example destination latitude + private static final double DST_LON = -118.2437; // Example destination longitude + + @Test + public void testFromString() { + assertEquals(GeoDistance.PLANE, GeoDistance.fromString("plane")); + assertEquals(GeoDistance.ARC, GeoDistance.fromString("arc")); + } + + @Test(expected = IllegalArgumentException.class) + public void testFromStringInvalid() { + GeoDistance.fromString("invalid"); + } + + @Test + public void testCalculatePlane() { + double expectedDistanceInMeters = 3978199.0100920075; + double actualDistance = GeoDistance.PLANE.calculate(SRC_LAT, SRC_LON, DST_LAT, DST_LON, DistanceUnit.METERS); + assertEquals(expectedDistanceInMeters, actualDistance, 0.01); + } + + @Test + public void testCalculateArc() { + double expectedDistanceInMeters = 3935751.673226063; + double actualDistance = GeoDistance.ARC.calculate(SRC_LAT, SRC_LON, DST_LAT, DST_LON, DistanceUnit.METERS); + assertEquals(expectedDistanceInMeters, actualDistance, 0.01); + } + + +} diff --git a/persistence-spi/src/test/java/org/apache/unomi/persistence/PropertyHelperTest.java b/persistence-spi/src/test/java/org/apache/unomi/persistence/spi/PropertyHelperTest.java similarity index 99% rename from persistence-spi/src/test/java/org/apache/unomi/persistence/PropertyHelperTest.java rename to persistence-spi/src/test/java/org/apache/unomi/persistence/spi/PropertyHelperTest.java index 3ee01b3958..b421e6777a 100644 --- a/persistence-spi/src/test/java/org/apache/unomi/persistence/PropertyHelperTest.java +++ b/persistence-spi/src/test/java/org/apache/unomi/persistence/spi/PropertyHelperTest.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.unomi.persistence; +package org.apache.unomi.persistence.spi; import org.apache.unomi.api.Profile; import org.apache.unomi.persistence.spi.PropertyHelper; diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionEvaluator.java index 1ecfc65be4..ade280e6d7 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionEvaluator.java @@ -19,8 +19,8 @@ import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; import java.util.List; import java.util.Map; diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java index a9be992035..b98ce9150d 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/GeoLocationByPointSessionConditionEvaluator.java @@ -20,10 +20,10 @@ import org.apache.commons.beanutils.BeanUtils; import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; -import org.elasticsearch.common.geo.GeoDistance; -import org.elasticsearch.common.unit.DistanceUnit; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.geo.DistanceUnit; +import org.apache.unomi.persistence.spi.conditions.geo.GeoDistance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/IdsConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/IdsConditionEvaluator.java index d4f1ca8879..32f5b9468d 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/IdsConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/IdsConditionEvaluator.java @@ -18,8 +18,8 @@ import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; import java.util.Collection; import java.util.Map; diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionEvaluator.java index ee8c1e8843..cab1c35feb 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/MatchAllConditionEvaluator.java @@ -19,8 +19,8 @@ import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; import java.util.Map; diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NestedConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NestedConditionEvaluator.java index be6074b359..2a6a112b15 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NestedConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NestedConditionEvaluator.java @@ -22,8 +22,8 @@ import org.apache.unomi.api.Profile; import org.apache.unomi.api.Session; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionEvaluator.java index 5522f3196b..9296214bbf 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/NotConditionEvaluator.java @@ -19,8 +19,8 @@ import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; import java.util.Map; diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java index f14f768f62..6d9ce91e18 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PastEventConditionEvaluator.java @@ -22,9 +22,10 @@ import org.apache.unomi.api.Profile; import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.services.DefinitionsService; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; import org.apache.unomi.persistence.spi.PersistenceService; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.PastEventConditionPersistenceQueryBuilder; import org.apache.unomi.scripting.ScriptExecutor; import java.util.ArrayList; @@ -36,6 +37,7 @@ public class PastEventConditionEvaluator implements ConditionEvaluator { private PersistenceService persistenceService; private DefinitionsService definitionsService; private ScriptExecutor scriptExecutor; + private PastEventConditionPersistenceQueryBuilder pastEventConditionPersistenceQueryBuilder; public void setPersistenceService(PersistenceService persistenceService) { this.persistenceService = persistenceService; @@ -45,6 +47,10 @@ public void setDefinitionsService(DefinitionsService definitionsService) { this.definitionsService = definitionsService; } + public void setPastEventConditionPersistenceQueryBuilder(PastEventConditionPersistenceQueryBuilder pastEventConditionPersistenceQueryBuilder) { + this.pastEventConditionPersistenceQueryBuilder = pastEventConditionPersistenceQueryBuilder; + } + public void setScriptExecutor(ScriptExecutor scriptExecutor) { this.scriptExecutor = scriptExecutor; } @@ -70,10 +76,10 @@ public boolean eval(Condition condition, Item item, Map context, } } else { // TODO see for deprecation, this should not happen anymore each past event condition should have a generatedPropertyKey - count = persistenceService.queryCount(PastEventConditionESQueryBuilder.getEventCondition(condition, context, item.getItemId(), definitionsService, scriptExecutor), Event.ITEM_TYPE); + count = persistenceService.queryCount(pastEventConditionPersistenceQueryBuilder.getEventCondition(condition, context, item.getItemId(), definitionsService, scriptExecutor), Event.ITEM_TYPE); } - boolean eventsOccurred = PastEventConditionESQueryBuilder.getStrategyFromOperator((String) condition.getParameter("operator")); + boolean eventsOccurred = pastEventConditionPersistenceQueryBuilder.getStrategyFromOperator((String) condition.getParameter("operator")); if (eventsOccurred) { int minimumEventCount = parameters.get("minimumEventCount") == null ? 0 : (Integer) parameters.get("minimumEventCount"); int maximumEventCount = parameters.get("maximumEventCount") == null ? Integer.MAX_VALUE : (Integer) parameters.get("maximumEventCount"); diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java deleted file mode 100644 index a7df6c7a40..0000000000 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionESQueryBuilder.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.unomi.plugins.baseplugin.conditions; - -import org.apache.commons.lang3.ObjectUtils; -import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionContextHelper; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.unit.DistanceUnit; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormatter; -import org.joda.time.format.ISODateTimeFormat; - -import java.util.*; - -import static org.apache.unomi.plugins.baseplugin.conditions.PropertyConditionEvaluator.getDate; - -public class PropertyConditionESQueryBuilder implements ConditionESQueryBuilder { - - DateTimeFormatter dateTimeFormatter; - - public PropertyConditionESQueryBuilder() { - dateTimeFormatter = ISODateTimeFormat.dateTime(); - } - - @Override - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { - String comparisonOperator = (String) condition.getParameter("comparisonOperator"); - String name = (String) condition.getParameter("propertyName"); - - if (comparisonOperator == null || name == null) { - throw new IllegalArgumentException("Impossible to build ES filter, condition is not valid, comparisonOperator and propertyName properties should be provided"); - } - - String expectedValue = ConditionContextHelper.foldToASCII((String) condition.getParameter("propertyValue")); - Object expectedValueInteger = condition.getParameter("propertyValueInteger"); - Object expectedValueDouble = condition.getParameter("propertyValueDouble"); - Object expectedValueDate = convertDateToISO(condition.getParameter("propertyValueDate")); - Object expectedValueDateExpr = condition.getParameter("propertyValueDateExpr"); - - Collection expectedValues = ConditionContextHelper.foldToASCII((Collection) condition.getParameter("propertyValues")); - Collection expectedValuesInteger = (Collection) condition.getParameter("propertyValuesInteger"); - Collection expectedValuesDouble = (Collection) condition.getParameter("propertyValuesDouble"); - Collection expectedValuesDate = convertDatesToISO((Collection) condition.getParameter("propertyValuesDate")); - Collection expectedValuesDateExpr = (Collection) condition.getParameter("propertyValuesDateExpr"); - - Object value = ObjectUtils.firstNonNull(expectedValue, expectedValueInteger, expectedValueDouble, expectedValueDate, expectedValueDateExpr); - @SuppressWarnings("unchecked") - Collection values = ObjectUtils.firstNonNull(expectedValues, expectedValuesInteger, expectedValuesDouble, expectedValuesDate, expectedValuesDateExpr); - - switch (comparisonOperator) { - case "equals": - checkRequiredValue(value, name, comparisonOperator, false); - return QueryBuilders.termQuery(name, value); - case "notEquals": - checkRequiredValue(value, name, comparisonOperator, false); - return QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(name, value)); - case "greaterThan": - checkRequiredValue(value, name, comparisonOperator, false); - return QueryBuilders.rangeQuery(name).gt(value); - case "greaterThanOrEqualTo": - checkRequiredValue(value, name, comparisonOperator, false); - return QueryBuilders.rangeQuery(name).gte(value); - case "lessThan": - checkRequiredValue(value, name, comparisonOperator, false); - return QueryBuilders.rangeQuery(name).lt(value); - case "lessThanOrEqualTo": - checkRequiredValue(value, name, comparisonOperator, false); - return QueryBuilders.rangeQuery(name).lte(value); - case "between": - checkRequiredValuesSize(values, name, comparisonOperator, 2); - Iterator iterator = values.iterator(); - return QueryBuilders.rangeQuery(name).gte(iterator.next()).lte(iterator.next()); - case "exists": - return QueryBuilders.existsQuery(name); - case "missing": - return QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery((name))); - case "contains": - checkRequiredValue(expectedValue, name, comparisonOperator, false); - return QueryBuilders.regexpQuery(name, ".*" + expectedValue + ".*"); - case "notContains": - checkRequiredValue(expectedValue, name, comparisonOperator, false); - return QueryBuilders.boolQuery().mustNot(QueryBuilders.regexpQuery(name, ".*" + expectedValue + ".*")); - case "startsWith": - checkRequiredValue(expectedValue, name, comparisonOperator, false); - return QueryBuilders.prefixQuery(name, expectedValue); - case "endsWith": - checkRequiredValue(expectedValue, name, comparisonOperator, false); - return QueryBuilders.regexpQuery(name, ".*" + expectedValue); - case "matchesRegex": - checkRequiredValue(expectedValue, name, comparisonOperator, false); - return QueryBuilders.regexpQuery(name, expectedValue); - case "in": - checkRequiredValue(values, name, comparisonOperator, true); - return QueryBuilders.termsQuery(name, values.toArray()); - case "notIn": - checkRequiredValue(values, name, comparisonOperator, true); - return QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(name, values.toArray())); - case "all": - checkRequiredValue(values, name, comparisonOperator, true); - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (Object curValue : values) { - boolQueryBuilder.must(QueryBuilders.termQuery(name, curValue)); - } - return boolQueryBuilder; - case "inContains": - checkRequiredValue(values, name, comparisonOperator, true); - BoolQueryBuilder boolQueryBuilderInContains = QueryBuilders.boolQuery(); - for (Object curValue : values) { - boolQueryBuilderInContains.must(QueryBuilders.regexpQuery(name, ".*" + curValue + ".*")); - } - return boolQueryBuilderInContains; - case "hasSomeOf": - checkRequiredValue(values, name, comparisonOperator, true); - boolQueryBuilder = QueryBuilders.boolQuery(); - for (Object curValue : values) { - boolQueryBuilder.should(QueryBuilders.termQuery(name, curValue)); - } - return boolQueryBuilder; - case "hasNoneOf": - checkRequiredValue(values, name, comparisonOperator, true); - boolQueryBuilder = QueryBuilders.boolQuery(); - for (Object curValue : values) { - boolQueryBuilder.mustNot(QueryBuilders.termQuery(name, curValue)); - } - return boolQueryBuilder; - case "isDay": - checkRequiredValue(value, name, comparisonOperator, false); - return getIsSameDayRange(getDate(value), name); - case "isNotDay": - checkRequiredValue(value, name, comparisonOperator, false); - return QueryBuilders.boolQuery().mustNot(getIsSameDayRange(getDate(value), name)); - case "distance": - final String unitString = (String) condition.getParameter("unit"); - final Object centerObj = condition.getParameter("center"); - final Double distance = (Double) condition.getParameter("distance"); - - if (centerObj != null && distance != null) { - String centerString; - if (centerObj instanceof org.apache.unomi.api.GeoPoint) { - centerString = ((org.apache.unomi.api.GeoPoint) centerObj).asString(); - } else if (centerObj instanceof String) { - centerString = (String) centerObj; - } else { - centerString = centerObj.toString(); - } - DistanceUnit unit = unitString != null ? DistanceUnit.fromString(unitString) : DistanceUnit.DEFAULT; - - return QueryBuilders.geoDistanceQuery(name) - .ignoreUnmapped(true) - .distance(distance, unit) - .point(new GeoPoint(centerString)); - } - } - return null; - } - - private void checkRequiredValuesSize(Collection values, String name, String operator, int expectedSize) { - if (values == null || values.size() != expectedSize) { - throw new IllegalArgumentException("Impossible to build ES filter, missing " + expectedSize + " values for a condition using comparisonOperator: " + operator + ", and propertyName: " + name); - } - } - - private void checkRequiredValue(Object value, String name, String operator, boolean multiple) { - if (value == null) { - throw new IllegalArgumentException("Impossible to build ES filter, missing value" + (multiple ? "s" : "") + " for condition using comparisonOperator: " + operator + ", and propertyName: " + name); - } - } - - private QueryBuilder getIsSameDayRange(Object value, String name) { - DateTime date = new DateTime(value); - DateTime dayStart = date.withTimeAtStartOfDay(); - DateTime dayAfterStart = date.plusDays(1).withTimeAtStartOfDay(); - return QueryBuilders.rangeQuery(name).gte(convertDateToISO(dayStart.toDate())).lte(convertDateToISO(dayAfterStart.toDate())); - } - - private Object convertDateToISO(Object dateValue) { - if (dateValue == null) { - return dateValue; - } - if (dateValue instanceof Date) { - return dateTimeFormatter.print(new DateTime(dateValue)); - } else { - return dateValue; - } - } - - private Collection convertDatesToISO(Collection datesValues) { - List results = new ArrayList<>(); - if (datesValues == null) { - return null; - } - for (Object dateValue : datesValues) { - if (dateValue != null) { - results.add(convertDateToISO(dateValue)); - } - } - return results; - } -} diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java index c128d8f2c1..0e1ecadc71 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/PropertyConditionEvaluator.java @@ -9,13 +9,13 @@ * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ -package org.apache.unomi.plugins.baseplugin.conditions; + package org.apache.unomi.plugins.baseplugin.conditions; import org.apache.commons.lang3.ObjectUtils; import org.apache.commons.lang3.StringUtils; @@ -23,17 +23,15 @@ import org.apache.unomi.api.GeoPoint; import org.apache.unomi.api.Item; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionContextHelper; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; import org.apache.unomi.persistence.spi.PropertyHelper; +import org.apache.unomi.persistence.spi.conditions.ConditionContextHelper; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.DateUtils; +import org.apache.unomi.persistence.spi.conditions.geo.DistanceUnit; import org.apache.unomi.plugins.baseplugin.conditions.accessors.HardcodedPropertyAccessor; import org.apache.unomi.scripting.ExpressionFilterFactory; import org.apache.unomi.scripting.SecureFilteringClassLoader; -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.joda.Joda; -import org.elasticsearch.common.joda.JodaDateMathParser; -import org.elasticsearch.common.unit.DistanceUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,6 +40,8 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; +import static org.apache.unomi.persistence.spi.conditions.DateUtils.getDate; + /** * Evaluator for property comparison conditions */ @@ -87,7 +87,7 @@ private int compare(Object actualValue, String expectedValue, Object expectedVal private boolean compareValues(Object actualValue, Collection expectedValues, Collection expectedValuesInteger, Collection expectedValuesDouble, Collection expectedValuesDate, Collection expectedValuesDateExpr, String op) { Collection expectedDateExpr = null; if (expectedValuesDateExpr != null) { - expectedDateExpr = expectedValuesDateExpr.stream().map(PropertyConditionEvaluator::getDate).collect(Collectors.toList()); + expectedDateExpr = expectedValuesDateExpr.stream().map(DateUtils::getDate).collect(Collectors.toList()); } @SuppressWarnings("unchecked") Collection expected = ObjectUtils.firstNonNull(expectedValues, expectedValuesDate, expectedValuesInteger, expectedValuesDouble, expectedDateExpr); @@ -316,24 +316,6 @@ protected Object getHardcodedPropertyValue(Item item, String expression) { return hardcodedPropertyAccessorRegistry.getProperty(item, expression); } - protected static Date getDate(Object value) { - if (value == null) { - return null; - } - if (value instanceof Date) { - return ((Date) value); - } else { - JodaDateMathParser parser = new JodaDateMathParser(Joda.forPattern("strictDateOptionalTime||epoch_millis")); - try { - return Date.from(parser.parse(value.toString(), System::currentTimeMillis)); - } catch (ElasticsearchParseException e) { - LOGGER.warn("unable to parse date. See debug log level for full stacktrace"); - LOGGER.debug("unable to parse date {}", value, e); - } - } - return null; - } - @SuppressWarnings("unchecked") private List getValueSet(Object expectedValue) { if (expectedValue instanceof List) { @@ -376,4 +358,4 @@ private Object getSecond(Collection collection) { return null; } -} +} \ No newline at end of file diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java deleted file mode 100644 index 7ba6c6f65d..0000000000 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionESQueryBuilder.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.unomi.plugins.baseplugin.conditions; - -import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -public class SourceEventPropertyConditionESQueryBuilder implements ConditionESQueryBuilder { - - public SourceEventPropertyConditionESQueryBuilder() { - } - - private void appendFilderIfPropExist(List queryBuilders, Condition condition, String prop){ - final Object parameter = condition.getParameter(prop); - if (parameter != null && !"".equals(parameter)) { - queryBuilders.add(QueryBuilders.termQuery("source." + prop, (String) parameter)); - } - } - - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { - List queryBuilders = new ArrayList(); - for (String prop : new String[]{"id", "path", "scope", "type"}){ - appendFilderIfPropExist(queryBuilders, condition, prop); - } - - if (queryBuilders.size() >= 1) { - if (queryBuilders.size() == 1) { - return queryBuilders.get(0); - } else { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryBuilder queryBuilder : queryBuilders) { - boolQueryBuilder.must(queryBuilder); - } - return boolQueryBuilder; - } - } else { - return null; - } - } -} \ No newline at end of file diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionEvaluator.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionEvaluator.java index 19bbca7c6a..5876cb379f 100644 --- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionEvaluator.java +++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/SourceEventPropertyConditionEvaluator.java @@ -22,8 +22,8 @@ import org.apache.unomi.api.conditions.Condition; import org.apache.unomi.api.conditions.ConditionType; import org.apache.unomi.api.services.DefinitionsService; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluator; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionEvaluatorDispatcher; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluator; +import org.apache.unomi.persistence.spi.conditions.ConditionEvaluatorDispatcher; import java.util.ArrayList; import java.util.HashMap; diff --git a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml index 33ec992223..d69f91c783 100644 --- a/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml +++ b/plugins/baseplugin/src/main/resources/OSGI-INF/blueprint/blueprint.xml @@ -50,106 +50,24 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - + - + - + @@ -159,27 +77,27 @@ - + - + - + - + @@ -188,18 +106,19 @@ - + + - + diff --git a/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java b/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java index 978edef2a9..7a6a7efe0b 100644 --- a/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java +++ b/plugins/hover-event/src/main/java/org/apache/unomi/plugins/events/hover/querybuilders/HoverEventConditionESQueryBuilder.java @@ -17,12 +17,11 @@ package org.apache.unomi.plugins.events.hover.querybuilders; +import co.elastic.clients.elasticsearch._types.query_dsl.Query; +import co.elastic.clients.elasticsearch._types.query_dsl.QueryBuilders; import org.apache.unomi.api.conditions.Condition; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilder; -import org.apache.unomi.persistence.elasticsearch.conditions.ConditionESQueryBuilderDispatcher; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilder; +import org.apache.unomi.persistence.elasticsearch.ConditionESQueryBuilderDispatcher; import java.util.ArrayList; import java.util.List; @@ -36,23 +35,19 @@ public class HoverEventConditionESQueryBuilder implements ConditionESQueryBuilde public HoverEventConditionESQueryBuilder() { } - public QueryBuilder buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { - List queryBuilders = new ArrayList(); - queryBuilders.add(QueryBuilders.termQuery("eventType", "hover")); + public Query buildQuery(Condition condition, Map context, ConditionESQueryBuilderDispatcher dispatcher) { + List queries = new ArrayList<>(); + queries.add(QueryBuilders.term(builder -> builder.field("eventType").value("hover"))); String targetId = (String) condition.getParameter("targetId"); String targetPath = (String) condition.getParameter("targetPath"); - if (targetId != null && targetId.trim().length() > 0) { - queryBuilders.add(QueryBuilders.termQuery("target.itemId", targetId)); + if (targetId != null && !targetId.trim().isEmpty()) { + queries.add(QueryBuilders.term(builder -> builder.field("target.itemId").value(targetId))); } else if (targetPath != null && targetPath.trim().length() > 0) { - queryBuilders.add(QueryBuilders.termQuery("target.properties.pageInfo.pagePath", targetPath)); + queries.add(QueryBuilders.term(builder -> builder.field("target.properties.pageInfo.pagePath").value(targetPath))); } else { - queryBuilders.add(QueryBuilders.termQuery("target.itemId", "")); + queries.add(QueryBuilders.term(builder -> builder.field("target.itemId").value(""))); } - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - for (QueryBuilder queryBuilder : queryBuilders) { - boolQueryBuilder.must(queryBuilder); - } - return boolQueryBuilder; + return QueryBuilders.bool().must(queries).build()._toQuery(); } } diff --git a/pom.xml b/pom.xml index d949d7c882..bfa0d0355b 100644 --- a/pom.xml +++ b/pom.xml @@ -64,8 +64,8 @@ ${java.version} 4.4.8 - 7.4.2 - 7.11.0 + 9.0.3 + 9.1.3 1.1.0.Final 3.18.0 2.20.0 @@ -76,8 +76,7 @@ 2.4 3.10 2.19.0 - 8.2.0 - 3.12.8 + 9.12.2 2.4.0 2.12.7 4.3.0 @@ -91,6 +90,7 @@ 20250517 3.0.2 3.0.0 + 2.1.2 5.27.1 3.6.5 2.1 @@ -374,10 +374,6 @@ - bom api diff --git a/rest/src/main/java/org/apache/unomi/rest/endpoints/ScopeServiceEndPoint.java b/rest/src/main/java/org/apache/unomi/rest/endpoints/ScopeServiceEndPoint.java index 6ec5a054a1..7e07b803d8 100644 --- a/rest/src/main/java/org/apache/unomi/rest/endpoints/ScopeServiceEndPoint.java +++ b/rest/src/main/java/org/apache/unomi/rest/endpoints/ScopeServiceEndPoint.java @@ -60,8 +60,7 @@ public void setScopeService(ScopeService scopeService) { */ @GET @Path("/") - public List getScopes() { - return scopeService.getScopes(); + public List getScopes() {return scopeService.getScopes(); } /** diff --git a/scripting/pom.xml b/scripting/pom.xml index 800f0481c4..57fc31cac0 100644 --- a/scripting/pom.xml +++ b/scripting/pom.xml @@ -65,12 +65,6 @@ commons-lang3 provided - - org.slf4j - slf4j-api - provided - - junit junit diff --git a/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java index 8fa7e1e687..db8e9468fc 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/definitions/DefinitionsServiceImpl.java @@ -487,7 +487,7 @@ public Condition extractConditionBySystemTag(Condition rootCondition, String sys if (rootCondition.containsParameter("subConditions")) { @SuppressWarnings("unchecked") List subConditions = (List) rootCondition.getParameter("subConditions"); - List matchingConditions = new ArrayList(); + List matchingConditions = new ArrayList<>(); for (Condition condition : subConditions) { Condition c = extractConditionBySystemTag(condition, systemTag); if (c != null) { diff --git a/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java index 2110bedc00..f345466c7d 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/events/EventServiceImpl.java @@ -192,75 +192,6 @@ private int send(Event event, int depth) { return changes; } - @Override - public List getEventProperties() { - Map> mappings = persistenceService.getPropertiesMapping(Event.ITEM_TYPE); - List props = new ArrayList<>(mappings.size()); - getEventProperties(mappings, props, ""); - return props; - } - - @SuppressWarnings("unchecked") - private void getEventProperties(Map> mappings, List props, String prefix) { - for (Map.Entry> e : mappings.entrySet()) { - if (e.getValue().get("properties") != null) { - getEventProperties((Map>) e.getValue().get("properties"), props, prefix + e.getKey() + "."); - } else { - props.add(new EventProperty(prefix + e.getKey(), (String) e.getValue().get("type"))); - } - } - } - - private List getEventPropertyTypes() { - Map> mappings = persistenceService.getPropertiesMapping(Event.ITEM_TYPE); - return new ArrayList<>(getEventPropertyTypes(mappings)); - } - - @SuppressWarnings("unchecked") - private Set getEventPropertyTypes(Map> mappings) { - Set properties = new LinkedHashSet<>(); - for (Map.Entry> e : mappings.entrySet()) { - Set childProperties = null; - Metadata propertyMetadata = new Metadata(null, e.getKey(), e.getKey(), null); - Set systemTags = new HashSet<>(); - propertyMetadata.setSystemTags(systemTags); - PropertyType propertyType = new PropertyType(propertyMetadata); - propertyType.setTarget("event"); - ValueType valueType = null; - if (e.getValue().get("properties") != null) { - childProperties = getEventPropertyTypes((Map>) e.getValue().get("properties")); - valueType = definitionsService.getValueType("set"); - if (childProperties != null && childProperties.size() > 0) { - propertyType.setChildPropertyTypes(childProperties); - } - } else { - valueType = mappingTypeToValueType( (String) e.getValue().get("type")); - } - propertyType.setValueTypeId(valueType.getId()); - propertyType.setValueType(valueType); - properties.add(propertyType); - } - return properties; - } - - private ValueType mappingTypeToValueType(String mappingType) { - if ("text".equals(mappingType)) { - return definitionsService.getValueType("string"); - } else if ("date".equals(mappingType)) { - return definitionsService.getValueType("date"); - } else if ("long".equals(mappingType)) { - return definitionsService.getValueType("integer"); - } else if ("boolean".equals(mappingType)) { - return definitionsService.getValueType("boolean"); - } else if ("set".equals(mappingType)) { - return definitionsService.getValueType("set"); - } else if ("object".equals(mappingType)) { - return definitionsService.getValueType("set"); - } else { - return definitionsService.getValueType("unknown"); - } - } - public Set getEventTypeIds() { Map dynamicEventTypeIds = persistenceService.aggregateWithOptimizedQuery(null, new TermsAggregate("eventType"), Event.ITEM_TYPE); Set eventTypeIds = new LinkedHashSet(predefinedEventTypeIds); diff --git a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java index ec9c6f0b70..95c9cda6db 100644 --- a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java +++ b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java @@ -438,7 +438,7 @@ public GoalReport getGoalReport(String goalId) { public GoalReport getGoalReport(String goalId, AggregateQuery query) { Condition condition = new Condition(definitionsService.getConditionType("booleanCondition")); - final ArrayList list = new ArrayList(); + final ArrayList list = new ArrayList<>(); condition.setParameter("operator", "and"); condition.setParameter("subConditions", list); @@ -479,17 +479,20 @@ public GoalReport getGoalReport(String goalId, AggregateQuery query) { String interval = (String) query.getAggregate().getParameters().get("interval"); String format = (String) query.getAggregate().getParameters().get("format"); aggregate = new DateAggregate(property, interval, format); - } else if (query.getAggregate().getType().equals("dateRange") && query.getAggregate().getDateRanges() != null && query.getAggregate().getDateRanges().size() > 0) { + } else if (query.getAggregate().getType().equals("dateRange") && query.getAggregate().getDateRanges() != null && !query.getAggregate() + .getDateRanges().isEmpty()) { String format = (String) query.getAggregate().getParameters().get("format"); aggregate = new DateRangeAggregate(property, format, query.getAggregate().getDateRanges()); - } else if (query.getAggregate().getType().equals("numericRange") && query.getAggregate().getNumericRanges() != null && query.getAggregate().getNumericRanges().size() > 0) { + } else if (query.getAggregate().getType().equals("numericRange") && query.getAggregate().getNumericRanges() != null && !query.getAggregate() + .getNumericRanges().isEmpty()) { aggregate = new NumericRangeAggregate(property, query.getAggregate().getNumericRanges()); - } else if (query.getAggregate().getType().equals("ipRange") && query.getAggregate().ipRanges() != null && query.getAggregate().ipRanges().size() > 0) { + } else if (query.getAggregate().getType().equals("ipRange") && query.getAggregate().ipRanges() != null && !query.getAggregate() + .ipRanges().isEmpty()) { aggregate = new IpRangeAggregate(property, query.getAggregate().ipRanges()); } } - if(aggregate == null){ + if (aggregate == null) { aggregate = new TermsAggregate(property); } } @@ -503,12 +506,12 @@ public GoalReport getGoalReport(String goalId, AggregateQuery query) { match = persistenceService.aggregateWithOptimizedQuery(condition, aggregate, Session.ITEM_TYPE); } else { list.add(goalStartCondition); - all = new HashMap(); + all = new HashMap<>(); all.put("_filtered", persistenceService.queryCount(condition, Session.ITEM_TYPE)); list.remove(goalStartCondition); list.add(goalTargetCondition); - match = new HashMap(); + match = new HashMap<>(); match.put("_filtered", persistenceService.queryCount(condition, Session.ITEM_TYPE)); } @@ -522,7 +525,7 @@ public GoalReport getGoalReport(String goalId, AggregateQuery query) { stat.setConversionRate(stat.getStartCount() > 0 ? (float) stat.getTargetCount() / (float) stat.getStartCount() : 0); report.setGlobalStats(stat); all.remove("_all"); - report.setSplit(new LinkedList()); + report.setSplit(new LinkedList<>()); for (Map.Entry entry : all.entrySet()) { GoalReport.Stat dateStat = new GoalReport.Stat(); dateStat.setKey(entry.getKey()); diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java index 4f2fc21c85..53cc90a947 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/migration/utils/MigrationUtils.java @@ -313,9 +313,9 @@ public static void waitForYellowStatus(CloseableHttpClient httpClient, String es /** * Updates documents in an index based on a specified query. - *

- * This method sends a request to update documents that match the provided query in the specified index. The update operation is - * performed asynchronously, and the method waits for the task to complete before returning. + * + *

This method sends a request to update documents that match the provided query in the specified index. The update operation is + * performed asynchronously, and the method waits for the task to complete before returning.

* * @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server * @param esAddress the address of the Elasticsearch server @@ -332,10 +332,10 @@ public static void updateByQuery(CloseableHttpClient httpClient, String esAddres /** * Deletes documents from an index based on a specified query. - *

- * This method sends a request to the Elasticsearch cluster to delete documents + * + *

This method sends a request to the Elasticsearch cluster to delete documents * that match the provided query in the specified index. The deletion operation is - * performed asynchronously, and the method waits for the task to complete before returning. + * performed asynchronously, and the method waits for the task to complete before returning.

* * @param httpClient the CloseableHttpClient used to send the request to the Elasticsearch server * @param esAddress the address of the Elasticsearch server diff --git a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java index fbab16cd3f..92fbf6a584 100644 --- a/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java +++ b/tools/shell-commands/src/main/java/org/apache/unomi/shell/services/internal/UnomiManagementServiceImpl.java @@ -106,6 +106,7 @@ public void initReversedBundleSymbolicNames() { bundleSymbolicNames.add("org.apache.unomi.metrics"); bundleSymbolicNames.add("org.apache.unomi.persistence-spi"); bundleSymbolicNames.add("org.apache.unomi.persistence-elasticsearch-core"); + bundleSymbolicNames.add("org.apache.unomi.persistence-elasticsearch-conditions"); bundleSymbolicNames.add("org.apache.unomi.services"); bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-services"); bundleSymbolicNames.add("org.apache.unomi.cxs-lists-extension-rest");