From 39966cd0dd977f97cae10ab4b86587a3f6f246c7 Mon Sep 17 00:00:00 2001 From: Lukas Bindreiter Date: Thu, 16 Jul 2026 11:32:26 +0200 Subject: [PATCH] Updated field annotations for queryables, roles, and json_path --- apis/datasets/v1/data_access.proto | 121 ++++++++++++++++++++++++++++ apis/datasets/v1/dataset_type.proto | 52 +++++++++--- apis/datasets/v1/datasets.proto | 4 +- 3 files changed, 166 insertions(+), 11 deletions(-) diff --git a/apis/datasets/v1/data_access.proto b/apis/datasets/v1/data_access.proto index 44ea19f..4f4d117 100644 --- a/apis/datasets/v1/data_access.proto +++ b/apis/datasets/v1/data_access.proto @@ -7,6 +7,8 @@ package datasets.v1; import "buf/validate/validate.proto"; import "datasets/v1/core.proto"; import "datasets/v1/well_known_types.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; import "tilebox/v1/id.proto"; import "tilebox/v1/query.proto"; @@ -41,6 +43,125 @@ message QueryFilters { tilebox.v1.IDInterval datapoint_interval = 2; SpatialFilter spatial_extent = 3; + + // Additional expressions over fields marked queryable in the dataset schema. + // All top-level expressions are combined with each other and with the interval and spatial filters using logical AND. + // Use a nested LogicalExpression for explicit OR, NOT, or grouped boolean logic. + repeated FilterExpression expressions = 4 [(buf.validate.field).repeated.max_items = 100]; +} + +// FilterExpression is a typed filter expression compatible with the Basic CQL2 property-to-literal subset. +message FilterExpression { + option (buf.validate.message).oneof = { + fields: [ + "logical", + "comparison", + "is_null" + ] + required: true + }; + + // Exactly one expression node must be set. + LogicalExpression logical = 1; + FieldComparison comparison = 2; + FieldNullCheck is_null = 3; +} + +// LogicalExpression combines nested filter expressions. +message LogicalExpression { + option (buf.validate.message).cel = { + id: "logical_expression.operand_arity" + message: "NOT requires exactly one operand; AND and OR require at least two" + expression: "this.operator == 3 ? this.operands.size() == 1 : this.operands.size() >= 2" + }; + + // The logical operation to apply to operands. + LogicalOperator operator = 1 [(buf.validate.field).enum = { + defined_only: true + not_in: [0] + }]; + // Operands for the logical operation. AND and OR require at least two; NOT requires exactly one. + repeated FilterExpression operands = 2 [(buf.validate.field).repeated = { + min_items: 1 + max_items: 100 + }]; +} + +// LogicalOperator specifies how nested filter expressions are combined. +enum LogicalOperator { + LOGICAL_OPERATOR_UNSPECIFIED = 0; + LOGICAL_OPERATOR_AND = 1; + LOGICAL_OPERATOR_OR = 2; + LOGICAL_OPERATOR_NOT = 3; +} + +// FieldComparison compares a queryable dataset field to a typed literal value. +message FieldComparison { + // Name of a top-level field in the dataset schema. + string field_name = 1 [(buf.validate.field).string = { + min_len: 1 + max_len: 100 + pattern: "^[a-z][a-z0-9_]*$" + }]; + // Comparison operation to apply. + FieldComparisonOperator operator = 2 [(buf.validate.field).enum = { + defined_only: true + not_in: [0] + }]; + // Literal value. Its kind must match the field's protobuf descriptor. + FieldQueryValue value = 3 [(buf.validate.field).required = true]; +} + +// FieldComparisonOperator specifies a Basic CQL2 comparison operation. +enum FieldComparisonOperator { + FIELD_COMPARISON_OPERATOR_UNSPECIFIED = 0; + FIELD_COMPARISON_OPERATOR_EQUAL = 1; + FIELD_COMPARISON_OPERATOR_NOT_EQUAL = 2; + FIELD_COMPARISON_OPERATOR_LESS_THAN = 3; + FIELD_COMPARISON_OPERATOR_LESS_THAN_OR_EQUAL = 4; + FIELD_COMPARISON_OPERATOR_GREATER_THAN = 5; + FIELD_COMPARISON_OPERATOR_GREATER_THAN_OR_EQUAL = 6; +} + +// FieldNullCheck matches datapoints for which a queryable field is absent or explicitly null. +// IS NOT NULL is represented by wrapping this expression in LOGICAL_OPERATOR_NOT. +message FieldNullCheck { + // Name of a top-level field in the dataset schema. + string field_name = 1 [(buf.validate.field).string = { + min_len: 1 + max_len: 100 + pattern: "^[a-z][a-z0-9_]*$" + }]; +} + +// FieldQueryValue is a typed scalar or well-known-type literal used in a field comparison. +message FieldQueryValue { + option (buf.validate.message).oneof = { + fields: [ + "bool_value", + "int64_value", + "uint64_value", + "double_value", + "string_value", + "timestamp_value", + "duration_value", + "enum_name", + "bytes_value" + ] + required: true + }; + + // Exactly one typed literal must be set. Explicit presence preserves zero and empty values. + bool bool_value = 1 [features.field_presence = EXPLICIT]; + int64 int64_value = 2 [features.field_presence = EXPLICIT]; + uint64 uint64_value = 3 [features.field_presence = EXPLICIT]; + double double_value = 4 [features.field_presence = EXPLICIT]; + string string_value = 5 [features.field_presence = EXPLICIT]; + google.protobuf.Timestamp timestamp_value = 6; + google.protobuf.Duration duration_value = 7; + // Symbolic value from the field's enum descriptor. + string enum_name = 8 [features.field_presence = EXPLICIT]; + bytes bytes_value = 9 [features.field_presence = EXPLICIT]; } // SpatialFilterMode specifies how geometries are compared to a given spatial filter. diff --git a/apis/datasets/v1/dataset_type.proto b/apis/datasets/v1/dataset_type.proto index e4677f2..1825c69 100644 --- a/apis/datasets/v1/dataset_type.proto +++ b/apis/datasets/v1/dataset_type.proto @@ -12,19 +12,19 @@ import "tilebox/v1/id.proto"; option features.field_presence = IMPLICIT; -// Field describes a field of a dataset. +// Field is the request representation of a dataset field. It intentionally combines only a protobuf field descriptor +// with its Tilebox annotation, matching the information represented separately by AnnotatedType. message Field { + reserved 3; + reserved queryable; + // The descriptor contains the name of the field, the type, optional labels (e.g. repeated) and other information. // If the type is TYPE_MESSAGE, then the type_name must be a fully qualified name to a well known type, e.g. // `datasets.v1.Vec3` or `google.protobuf.Timestamp`. google.protobuf.FieldDescriptorProto descriptor = 1 [(buf.validate.field).required = true]; - // An optional description and example value for the field. + // Additional documentation, source mapping, and query metadata for the field. FieldAnnotation annotation = 2; - - // A flag indicating whether the field should be queryable. This means we will build an index for the field, and - // allow users to query for certain values server-side. - bool queryable = 3 [(buf.validate.field).bool.const = false]; } // DatasetKind is an enum describing the kind of dataset. A dataset kind specifies a set of default fields, that @@ -40,13 +40,44 @@ enum DatasetKind { DATASET_KIND_SPATIOTEMPORAL = 2; } -// FieldAnnotation contains additional information about a field in a dataset +// FieldRole describes a semantic display role fulfilled by a dataset field. +// Thumbnail discovery is intentionally not represented here: thumbnails are assets whose STAC asset roles include +// thumbnail. +enum FieldRole { + FIELD_ROLE_UNSPECIFIED = 0; + + // The primary human-readable title or name used when displaying a datapoint. + FIELD_ROLE_PRIMARY_TITLE = 1; +} + +// FieldAnnotation contains additional information about a field in a dataset. message FieldAnnotation { string description = 1; string example_value = 2; + + // RFC 6901 JSON Pointer locating the field value in the source document. + // This allows flattened dataset fields to be reconstructed without relying on their protobuf field names. + string source_json_pointer = 3 [features.field_presence = EXPLICIT]; + + // Whether the field is projected into query storage and may be used in server-side filter expressions. + // Queryable fields are not necessarily backed by a secondary database index. + bool queryable = 4; + + // Optional JSON Schema reference to emit as `$ref` when advertising this field as a queryable. + // For example, this may reference the canonical STAC extension schema definition for a property. + string queryable_json_schema_ref = 5 [features.field_presence = EXPLICIT]; + + // Semantic display roles fulfilled by this field. + repeated FieldRole roles = 6 [ + (buf.validate.field).repeated.unique = true, + (buf.validate.field).repeated.items.enum = { + defined_only: true + not_in: [0] + } + ]; } -// DatasetType describes the type of a dataset. +// DatasetType is the request representation of a dataset type, used when creating or updating a dataset. message DatasetType { // kind denotes the kind of dataset this type describes. We do not rely on the default fields to be set in our // array of fields - since that way users could circumvent those by manually sending requests not from our Console. @@ -59,12 +90,15 @@ message DatasetType { repeated Field fields = 2; } -// AnnotatedType describes a message type +// AnnotatedType is the resolved representation returned on Dataset messages, including GetDataset responses. +// It combines the generated protobuf descriptors with the corresponding Tilebox field annotations. message AnnotatedType { google.protobuf.FileDescriptorSet descriptor_set = 1; // the url of the type, one of the types defined in the descriptor string type_url = 2; reserved 3; + // Annotations corresponding by index to the FieldDescriptorProto entries of the message selected by type_url. + // Empty annotations must be retained so this positional mapping remains stable. repeated FieldAnnotation field_annotations = 4; DatasetKind kind = 5; } diff --git a/apis/datasets/v1/datasets.proto b/apis/datasets/v1/datasets.proto index a3ce5fd..7bd4332 100644 --- a/apis/datasets/v1/datasets.proto +++ b/apis/datasets/v1/datasets.proto @@ -19,7 +19,7 @@ message CreateDatasetRequest { (buf.validate.field).string.min_len = 1, (buf.validate.field).string.max_len = 100 ]; - // message type of the dataset to create. + // Message type of the dataset to create, including per-field source mappings, queryable metadata, and roles. DatasetType type = 2 [(buf.validate.field).required = true]; // short text summary of the dataset to create. string summary = 3 [(buf.validate.field).string.max_len = 50000]; @@ -62,7 +62,7 @@ message UpdateDatasetRequest { (buf.validate.field).string.min_len = 1, (buf.validate.field).string.max_len = 100 ]; - // updated type of the dataset. + // Updated type of the dataset, including per-field source mappings, queryable metadata, and roles. DatasetType type = 3 [features.field_presence = EXPLICIT]; // updated summary of the dataset. string summary = 4 [