diff --git a/core/src/main/java/org/apache/iceberg/BaseTable.java b/core/src/main/java/org/apache/iceberg/BaseTable.java index 4d73c96fec28..858d0bf8b7ca 100644 --- a/core/src/main/java/org/apache/iceberg/BaseTable.java +++ b/core/src/main/java/org/apache/iceberg/BaseTable.java @@ -29,6 +29,7 @@ import org.apache.iceberg.metrics.MetricsReporter; import org.apache.iceberg.metrics.MetricsReporters; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.rest.labels.Labels; /** * Base {@link Table} implementation. @@ -40,20 +41,34 @@ * when reading the table data after deserialization. */ public class BaseTable - implements Table, HasTableOperations, Serializable, SupportsDistributedScanPlanning { + implements Table, + HasTableOperations, + Serializable, + SupportsDistributedScanPlanning, + SupportsLabels { private final TableOperations ops; private final String name; private MetricsReporter reporter; + // Ephemeral catalog enrichment, not table state. Transient because it is not preserved across + // serialization: writeReplace() swaps in a SerializableTable for Java serialization, and any + // path that bypasses writeReplace (e.g. Kryo) leaves this null, handled by labels(). + private final transient Labels labels; public BaseTable(TableOperations ops, String name) { this(ops, name, LoggingMetricsReporter.instance()); } public BaseTable(TableOperations ops, String name, MetricsReporter reporter) { + this(ops, name, reporter, Labels.empty()); + } + + public BaseTable(TableOperations ops, String name, MetricsReporter reporter, Labels labels) { Preconditions.checkNotNull(reporter, "reporter cannot be null"); + Preconditions.checkNotNull(labels, "labels cannot be null"); this.ops = ops; this.name = name; this.reporter = reporter; + this.labels = labels; } public MetricsReporter reporter() { @@ -74,6 +89,12 @@ public String name() { return name; } + @Override + public Labels labels() { + // labels is null only when this instance was deserialized bypassing writeReplace (e.g. Kryo). + return labels != null ? labels : Labels.empty(); + } + @Override public void refresh() { ops.refresh(); diff --git a/core/src/main/java/org/apache/iceberg/LabelsTable.java b/core/src/main/java/org/apache/iceberg/LabelsTable.java new file mode 100644 index 000000000000..dbf78a92af69 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/LabelsTable.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import java.util.List; +import java.util.Map; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.rest.labels.FieldLabels; +import org.apache.iceberg.rest.labels.Labels; +import org.apache.iceberg.types.Types; + +/** + * A {@link Table} implementation that exposes the catalog-provided labels of its base table as + * rows. + * + *

Each label key-value pair is a row. Object-level labels have {@code scope = "object"} and a + * null {@code field_id}; field-level labels have {@code scope = "field"} and the field id they are + * attached to. Labels are catalog-provided enrichment obtained at load time; a re-scan re-reads the + * base table's labels and may observe different values. + */ +public class LabelsTable extends BaseMetadataTable { + private static final String OBJECT_SCOPE = "object"; + private static final String FIELD_SCOPE = "field"; + + private static final Schema LABELS_SCHEMA = + new Schema( + Types.NestedField.required(1, "scope", Types.StringType.get()), + Types.NestedField.optional(2, "field_id", Types.IntegerType.get()), + Types.NestedField.required(3, "key", Types.StringType.get()), + Types.NestedField.required(4, "value", Types.StringType.get())); + + LabelsTable(Table table) { + this(table, table.name() + ".labels"); + } + + LabelsTable(Table table, String name) { + super(table, name); + } + + @Override + MetadataTableType metadataTableType() { + return MetadataTableType.LABELS; + } + + @Override + public TableScan newScan() { + return new LabelsScan(table()); + } + + @Override + public Schema schema() { + return LABELS_SCHEMA; + } + + private Labels labels() { + return table() instanceof SupportsLabels + ? ((SupportsLabels) table()).labels() + : Labels.empty(); + } + + private DataTask task(TableScan scan) { + Labels labels = labels(); + List rows = Lists.newArrayList(); + for (Map.Entry entry : labels.object().entrySet()) { + rows.add(StaticDataTask.Row.of(OBJECT_SCOPE, null, entry.getKey(), entry.getValue())); + } + + for (FieldLabels fieldLabels : labels.fields()) { + for (Map.Entry entry : fieldLabels.labels().entrySet()) { + rows.add( + StaticDataTask.Row.of( + FIELD_SCOPE, fieldLabels.fieldId(), entry.getKey(), entry.getValue())); + } + } + + return StaticDataTask.of( + table().io().newInputFile(table().operations().current().metadataFileLocation()), + schema(), + scan.schema(), + rows, + row -> row); + } + + private class LabelsScan extends StaticTableScan { + LabelsScan(Table table) { + super(table, LABELS_SCHEMA, MetadataTableType.LABELS, LabelsTable.this::task); + } + + LabelsScan(Table table, TableScanContext context) { + super(table, LABELS_SCHEMA, MetadataTableType.LABELS, LabelsTable.this::task, context); + } + + @Override + protected TableScan newRefinedScan(Table table, Schema schema, TableScanContext context) { + return new LabelsScan(table, context); + } + + @Override + public CloseableIterable planFiles() { + return CloseableIterable.withNoopClose(LabelsTable.this.task(this)); + } + } +} diff --git a/core/src/main/java/org/apache/iceberg/MetadataTableType.java b/core/src/main/java/org/apache/iceberg/MetadataTableType.java index 733a11d900f1..c5355df9adc4 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataTableType.java +++ b/core/src/main/java/org/apache/iceberg/MetadataTableType.java @@ -36,7 +36,8 @@ public enum MetadataTableType { ALL_FILES, ALL_MANIFESTS, ALL_ENTRIES, - POSITION_DELETES; + POSITION_DELETES, + LABELS; public static MetadataTableType from(String name) { try { diff --git a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java index adb0f18ba1ad..aa63a5f9794a 100644 --- a/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java +++ b/core/src/main/java/org/apache/iceberg/MetadataTableUtils.java @@ -88,6 +88,8 @@ private static Table createMetadataTableInstance( return new AllEntriesTable(baseTable, metadataTableName); case POSITION_DELETES: return new PositionDeletesTable(baseTable, metadataTableName); + case LABELS: + return new LabelsTable(baseTable, metadataTableName); default: throw new NoSuchTableException( "Unknown metadata table type: %s for %s", type, metadataTableName); diff --git a/core/src/main/java/org/apache/iceberg/SupportsLabels.java b/core/src/main/java/org/apache/iceberg/SupportsLabels.java new file mode 100644 index 000000000000..b9c534480155 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/SupportsLabels.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import org.apache.iceberg.rest.labels.Labels; + +/** + * Implemented by tables that can expose catalog-provided labels obtained from the load response. + * + *

Labels are optional, catalog-provided metadata enrichment. They are not part of table state + * and are not preserved when the table is serialized. + */ +public interface SupportsLabels { + /** + * Returns the catalog-provided labels for this table, or an empty instance when there are none. + */ + Labels labels(); +} diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index 1d8e9da6d43c..5fbb1da40baa 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -79,6 +79,7 @@ import org.apache.iceberg.rest.auth.AuthManagers; import org.apache.iceberg.rest.auth.AuthSession; import org.apache.iceberg.rest.credentials.Credential; +import org.apache.iceberg.rest.labels.Labels; import org.apache.iceberg.rest.requests.CommitTransactionRequest; import org.apache.iceberg.rest.requests.CreateNamespaceRequest; import org.apache.iceberg.rest.requests.CreateTableRequest; @@ -557,10 +558,11 @@ public Table loadTable(SessionContext context, TableIdentifier identifier) { } List credentials = response.credentials(); + Labels labels = response.labels(); RESTClient tableClient = client.withAuthSession(tableSession); Supplier tableSupplier = createTableSupplier( - finalIdentifier, tableMetadata, context, tableClient, tableConf, credentials); + finalIdentifier, tableMetadata, context, tableClient, tableConf, credentials, labels); String eTag = responseHeaders.getOrDefault(HttpHeaders.ETAG, null); if (eTag != null) { @@ -580,7 +582,8 @@ private Supplier createTableSupplier( SessionContext context, RESTClient tableClient, Map tableConf, - List credentials) { + List credentials, + Labels labels) { return () -> { RESTTableOperations ops = newTableOps( @@ -594,13 +597,16 @@ private Supplier createTableSupplier( trackFileIO(ops); - RESTTable table = restTableForScanPlanning(ops, identifier, tableClient, tableConf); + RESTTable table = restTableForScanPlanning(ops, identifier, tableClient, tableConf, labels); if (table != null) { return table; } return new BaseTable( - ops, fullTableName(identifier), metricsReporter(paths.metrics(identifier), tableClient)); + ops, + fullTableName(identifier), + metricsReporter(paths.metrics(identifier), tableClient), + labels); }; } @@ -608,7 +614,8 @@ private RESTTable restTableForScanPlanning( TableOperations ops, TableIdentifier finalIdentifier, RESTClient restClient, - Map tableConf) { + Map tableConf, + Labels labels) { String planningModeServerConfig = tableConf.get(RESTCatalogProperties.SCAN_PLANNING_MODE); ScanPlanningMode serverScanPlanningMode = planningModeServerConfig == null @@ -653,7 +660,8 @@ private RESTTable restTableForScanPlanning( paths, endpoints, properties(), - conf); + conf, + labels); } // Default to client-side planning @@ -740,13 +748,17 @@ public Table registerTable( trackFileIO(ops); - RESTTable restTable = restTableForScanPlanning(ops, ident, tableClient, tableConf); + RESTTable restTable = + restTableForScanPlanning(ops, ident, tableClient, tableConf, response.labels()); if (restTable != null) { return restTable; } return new BaseTable( - ops, fullTableName(ident), metricsReporter(paths.metrics(ident), tableClient)); + ops, + fullTableName(ident), + metricsReporter(paths.metrics(ident), tableClient), + response.labels()); } @Override @@ -1009,13 +1021,17 @@ public Table create() { trackFileIO(ops); - RESTTable restTable = restTableForScanPlanning(ops, ident, tableClient, tableConf); + RESTTable restTable = + restTableForScanPlanning(ops, ident, tableClient, tableConf, response.labels()); if (restTable != null) { return restTable; } return new BaseTable( - ops, fullTableName(ident), metricsReporter(paths.metrics(ident), tableClient)); + ops, + fullTableName(ident), + metricsReporter(paths.metrics(ident), tableClient), + response.labels()); } @Override diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTTable.java b/core/src/main/java/org/apache/iceberg/rest/RESTTable.java index 21d84d847781..e8501aff0b0a 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTTable.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTTable.java @@ -30,6 +30,7 @@ import org.apache.iceberg.TableScan; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.metrics.MetricsReporter; +import org.apache.iceberg.rest.labels.Labels; class RESTTable extends BaseTable implements SupportsDistributedScanPlanning { private final RESTClient client; @@ -51,8 +52,9 @@ class RESTTable extends BaseTable implements SupportsDistributedScanPlanning { ResourcePaths resourcePaths, Set supportedEndpoints, Map catalogProperties, - Object hadoopConf) { - super(ops, name, reporter); + Object hadoopConf, + Labels labels) { + super(ops, name, reporter, labels); this.reporter = reporter; this.client = client; this.headers = headers; diff --git a/core/src/main/java/org/apache/iceberg/rest/labels/FieldLabels.java b/core/src/main/java/org/apache/iceberg/rest/labels/FieldLabels.java new file mode 100644 index 000000000000..b6636a8cab90 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/labels/FieldLabels.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.labels; + +import java.util.Map; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.immutables.value.Value; + +/** Labels attached to a single field, identified by its field id. See {@link Labels}. */ +@Value.Immutable +public interface FieldLabels { + int fieldId(); + + Map labels(); + + @Value.Check + default void validate() { + Preconditions.checkArgument(fieldId() >= 1, "Invalid field id: must be >= 1"); + Preconditions.checkArgument(!labels().isEmpty(), "Invalid labels: must be non-empty"); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/labels/FieldLabelsParser.java b/core/src/main/java/org/apache/iceberg/rest/labels/FieldLabelsParser.java new file mode 100644 index 000000000000..6bf7b847818a --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/labels/FieldLabelsParser.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.labels; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; + +public class FieldLabelsParser { + private static final String FIELD_ID = "field-id"; + private static final String LABELS = "labels"; + + private FieldLabelsParser() {} + + public static String toJson(FieldLabels fieldLabels) { + return toJson(fieldLabels, false); + } + + public static String toJson(FieldLabels fieldLabels, boolean pretty) { + return JsonUtil.generate(gen -> toJson(fieldLabels, gen), pretty); + } + + public static void toJson(FieldLabels fieldLabels, JsonGenerator gen) throws IOException { + Preconditions.checkArgument(null != fieldLabels, "Invalid field labels: null"); + + gen.writeStartObject(); + + gen.writeNumberField(FIELD_ID, fieldLabels.fieldId()); + JsonUtil.writeStringMap(LABELS, fieldLabels.labels(), gen); + + gen.writeEndObject(); + } + + public static FieldLabels fromJson(String json) { + return JsonUtil.parse(json, FieldLabelsParser::fromJson); + } + + public static FieldLabels fromJson(JsonNode json) { + Preconditions.checkArgument(null != json, "Cannot parse field labels from null object"); + + return ImmutableFieldLabels.builder() + .fieldId(JsonUtil.getInt(FIELD_ID, json)) + .labels(JsonUtil.getStringMap(LABELS, json)) + .build(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/labels/Labels.java b/core/src/main/java/org/apache/iceberg/rest/labels/Labels.java new file mode 100644 index 000000000000..40cf73da47b4 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/labels/Labels.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.labels; + +import java.util.List; +import java.util.Map; +import org.immutables.value.Value; + +/** Optional catalog-provided labels returned on a load response. */ +@Value.Immutable +public interface Labels { + Labels EMPTY = ImmutableLabels.builder().build(); + + /** Object-level labels. */ + Map object(); + + /** Field-level labels, keyed by field id. */ + List fields(); + + /** Returns true when there are neither object-level nor field-level labels. */ + default boolean isEmpty() { + return object().isEmpty() && fields().isEmpty(); + } + + /** Returns a shared empty instance. */ + static Labels empty() { + return EMPTY; + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/labels/LabelsParser.java b/core/src/main/java/org/apache/iceberg/rest/labels/LabelsParser.java new file mode 100644 index 000000000000..3bc18a9d1bf2 --- /dev/null +++ b/core/src/main/java/org/apache/iceberg/rest/labels/LabelsParser.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.labels; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.util.JsonUtil; + +public class LabelsParser { + private static final String OBJECT = "object"; + private static final String FIELDS = "fields"; + + private LabelsParser() {} + + public static String toJson(Labels labels) { + return toJson(labels, false); + } + + public static String toJson(Labels labels, boolean pretty) { + return JsonUtil.generate(gen -> toJson(labels, gen), pretty); + } + + public static void toJson(Labels labels, JsonGenerator gen) throws IOException { + Preconditions.checkArgument(null != labels, "Invalid labels: null"); + + gen.writeStartObject(); + + if (!labels.object().isEmpty()) { + JsonUtil.writeStringMap(OBJECT, labels.object(), gen); + } + + if (!labels.fields().isEmpty()) { + gen.writeArrayFieldStart(FIELDS); + for (FieldLabels fieldLabels : labels.fields()) { + FieldLabelsParser.toJson(fieldLabels, gen); + } + + gen.writeEndArray(); + } + + gen.writeEndObject(); + } + + public static Labels fromJson(String json) { + return JsonUtil.parse(json, LabelsParser::fromJson); + } + + public static Labels fromJson(JsonNode json) { + Preconditions.checkArgument(null != json, "Cannot parse labels from null object"); + + ImmutableLabels.Builder builder = ImmutableLabels.builder(); + + if (json.hasNonNull(OBJECT)) { + builder.object(JsonUtil.getStringMap(OBJECT, json)); + } + + if (json.hasNonNull(FIELDS)) { + builder.fields(JsonUtil.getObjectList(FIELDS, json, FieldLabelsParser::fromJson)); + } + + return builder.build(); + } +} diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponse.java index 977220e7d782..fc754698109c 100644 --- a/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponse.java +++ b/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponse.java @@ -29,6 +29,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.rest.RESTResponse; import org.apache.iceberg.rest.credentials.Credential; +import org.apache.iceberg.rest.labels.Labels; /** * A REST response that is used when a table is successfully loaded. @@ -45,6 +46,7 @@ public class LoadTableResponse implements RESTResponse { private Map config; private TableMetadata metadataWithLocation; private List credentials; + private Labels labels; public LoadTableResponse() { // Required for Jackson deserialization @@ -54,11 +56,13 @@ private LoadTableResponse( String metadataLocation, TableMetadata metadata, Map config, - List credentials) { + List credentials, + Labels labels) { this.metadataLocation = metadataLocation; this.metadata = metadata; this.config = config; this.credentials = credentials; + this.labels = labels; } @Override @@ -87,12 +91,18 @@ public List credentials() { return credentials != null ? credentials : ImmutableList.of(); } + /** Catalog-provided labels, or an empty instance when the response carries no labels. */ + public Labels labels() { + return labels != null ? labels : Labels.empty(); + } + @Override public String toString() { return MoreObjects.toStringHelper(this) .add("metadataLocation", metadataLocation) .add("metadata", metadata) .add("config", config) + .add("labels", labels) .toString(); } @@ -105,6 +115,7 @@ public static class Builder { private TableMetadata metadata; private final Map config = Maps.newHashMap(); private final List credentials = Lists.newArrayList(); + private Labels labels; private Builder() {} @@ -134,9 +145,14 @@ public Builder addAllCredentials(List credentialsToAdd) { return this; } + public Builder withLabels(Labels tableLabels) { + this.labels = tableLabels; + return this; + } + public LoadTableResponse build() { Preconditions.checkNotNull(metadata, "Invalid metadata: null"); - return new LoadTableResponse(metadataLocation, metadata, config, credentials); + return new LoadTableResponse(metadataLocation, metadata, config, credentials, labels); } } } diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponseParser.java index 8d34b1498369..858df5fb0c2e 100644 --- a/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponseParser.java +++ b/core/src/main/java/org/apache/iceberg/rest/responses/LoadTableResponseParser.java @@ -26,6 +26,7 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.rest.credentials.Credential; import org.apache.iceberg.rest.credentials.CredentialParser; +import org.apache.iceberg.rest.labels.LabelsParser; import org.apache.iceberg.util.JsonUtil; public class LoadTableResponseParser { @@ -34,6 +35,7 @@ public class LoadTableResponseParser { private static final String METADATA = "metadata"; private static final String CONFIG = "config"; private static final String STORAGE_CREDENTIALS = "storage-credentials"; + private static final String LABELS = "labels"; private LoadTableResponseParser() {} @@ -70,6 +72,11 @@ public static void toJson(LoadTableResponse response, JsonGenerator gen) throws gen.writeEndArray(); } + if (!response.labels().isEmpty()) { + gen.writeFieldName(LABELS); + LabelsParser.toJson(response.labels(), gen); + } + gen.writeEndObject(); } @@ -101,6 +108,10 @@ public static LoadTableResponse fromJson(JsonNode json) { builder.addAllCredentials(LoadCredentialsResponseParser.fromJson(json).credentials()); } + if (json.hasNonNull(LABELS)) { + builder.withLabels(LabelsParser.fromJson(JsonUtil.get(LABELS, json))); + } + return builder.build(); } } diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponse.java b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponse.java index d07ba872fdaa..389ce5b19cc3 100644 --- a/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponse.java +++ b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponse.java @@ -20,6 +20,7 @@ import java.util.Map; import org.apache.iceberg.rest.RESTResponse; +import org.apache.iceberg.rest.labels.Labels; import org.apache.iceberg.view.ViewMetadata; import org.immutables.value.Value; @@ -31,6 +32,12 @@ public interface LoadViewResponse extends RESTResponse { Map config(); + /** Catalog-provided labels, or an empty instance when the response carries no labels. */ + @Value.Default + default Labels labels() { + return Labels.empty(); + } + @Override default void validate() { // nothing to validate as it's not possible to create an invalid instance diff --git a/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponseParser.java b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponseParser.java index a8aaf17e5d76..a8dcc9d1b737 100644 --- a/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponseParser.java +++ b/core/src/main/java/org/apache/iceberg/rest/responses/LoadViewResponseParser.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode; import java.io.IOException; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.rest.labels.LabelsParser; import org.apache.iceberg.util.JsonUtil; import org.apache.iceberg.view.ViewMetadata; import org.apache.iceberg.view.ViewMetadataParser; @@ -31,6 +32,7 @@ public class LoadViewResponseParser { private static final String METADATA_LOCATION = "metadata-location"; private static final String METADATA = "metadata"; private static final String CONFIG = "config"; + private static final String LABELS = "labels"; private LoadViewResponseParser() {} @@ -56,6 +58,11 @@ public static void toJson(LoadViewResponse response, JsonGenerator gen) throws I JsonUtil.writeStringMap(CONFIG, response.config(), gen); } + if (!response.labels().isEmpty()) { + gen.writeFieldName(LABELS); + LabelsParser.toJson(response.labels(), gen); + } + gen.writeEndObject(); } @@ -80,6 +87,10 @@ public static LoadViewResponse fromJson(JsonNode json) { builder.config(JsonUtil.getStringMap(CONFIG, json)); } + if (json.hasNonNull(LABELS)) { + builder.labels(LabelsParser.fromJson(JsonUtil.get(LABELS, json))); + } + return builder.build(); } } diff --git a/core/src/test/java/org/apache/iceberg/TestBaseTableLabels.java b/core/src/test/java/org/apache/iceberg/TestBaseTableLabels.java new file mode 100644 index 000000000000..83e32d2fdbeb --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestBaseTableLabels.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import org.apache.iceberg.metrics.LoggingMetricsReporter; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.labels.ImmutableFieldLabels; +import org.apache.iceberg.rest.labels.ImmutableLabels; +import org.apache.iceberg.rest.labels.Labels; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TestBaseTableLabels { + + private static final String TABLE_NAME = "tbl"; + + @TempDir private File temp; + + // Only the labels accessor is exercised here, not table operations, so no metadata is set up. + @Test + public void labelsDefaultToEmptyWhenNotProvided() { + BaseTable table = + new BaseTable(new TestTables.TestTableOperations(TABLE_NAME, temp), TABLE_NAME); + + assertThat(table).isInstanceOf(SupportsLabels.class); + assertThat(table.labels().isEmpty()).isTrue(); + } + + @Test + public void labelsAreExposedWhenProvided() { + Labels labels = + ImmutableLabels.builder() + .object(ImmutableMap.of("owner", "team-a")) + .addFields( + ImmutableFieldLabels.builder() + .fieldId(1) + .labels(ImmutableMap.of("classification", "pii")) + .build()) + .build(); + + BaseTable table = + new BaseTable( + new TestTables.TestTableOperations(TABLE_NAME, temp), + TABLE_NAME, + LoggingMetricsReporter.instance(), + labels); + + assertThat(table.labels()).isEqualTo(labels); + assertThat(table.labels().object()).containsEntry("owner", "team-a"); + assertThat(table.labels().fields()).hasSize(1); + } +} diff --git a/core/src/test/java/org/apache/iceberg/TestLabelsTable.java b/core/src/test/java/org/apache/iceberg/TestLabelsTable.java new file mode 100644 index 000000000000..f7ba21820fc4 --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/TestLabelsTable.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.File; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +public class TestLabelsTable { + + private static final HadoopTables TABLES = new HadoopTables(new Configuration()); + private static final Schema SCHEMA = + new Schema( + Types.NestedField.required(1, "id", Types.LongType.get()), + Types.NestedField.optional(2, "data", Types.StringType.get())); + + @TempDir private File tableDir; + private Table table; + + @BeforeEach + public void createTable() { + this.table = + TABLES.create( + SCHEMA, PartitionSpec.unpartitioned(), Maps.newHashMap(), tableDir.toURI().toString()); + } + + @Test + public void labelsTableResolvesThroughMetadataTableUtils() { + Table labelsTable = + MetadataTableUtils.createMetadataTableInstance(table, MetadataTableType.LABELS); + + assertThat(labelsTable).isInstanceOf(LabelsTable.class); + assertThat(labelsTable.schema().findField("scope")).isNotNull(); + assertThat(labelsTable.schema().findField("field_id")).isNotNull(); + assertThat(labelsTable.schema().findField("key")).isNotNull(); + assertThat(labelsTable.schema().findField("value")).isNotNull(); + } + + @Test + public void scanYieldsNoRowsWhenNoLabels() throws Exception { + Table labelsTable = new LabelsTable(table); + + List rows = ImmutableList.of(); + try (CloseableIterable tasks = labelsTable.newScan().planFiles()) { + for (FileScanTask task : tasks) { + try (CloseableIterable taskRows = task.asDataTask().rows()) { + rows = ImmutableList.copyOf(taskRows); + } + } + } + + // a HadoopTables table carries no catalog-provided labels + assertThat(rows).isEmpty(); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/labels/TestFieldLabelsParser.java b/core/src/test/java/org/apache/iceberg/rest/labels/TestFieldLabelsParser.java new file mode 100644 index 000000000000..1394c95f1dce --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/labels/TestFieldLabelsParser.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.labels; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +public class TestFieldLabelsParser { + + @Test + public void nullCheck() { + assertThatThrownBy(() -> FieldLabelsParser.toJson(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid field labels: null"); + + assertThatThrownBy(() -> FieldLabelsParser.fromJson((JsonNode) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse field labels from null object"); + } + + @Test + public void roundTrip() { + FieldLabels fieldLabels = + ImmutableFieldLabels.builder().fieldId(3).labels(ImmutableMap.of("pii", "true")).build(); + + String expectedJson = + """ + { + "field-id" : 3, + "labels" : { + "pii" : "true" + } + }"""; + + assertThat(FieldLabelsParser.toJson(fieldLabels, true)).isEqualTo(expectedJson); + assertThat(FieldLabelsParser.fromJson(expectedJson)).isEqualTo(fieldLabels); + } + + @Test + public void invalidFieldId() { + assertThatThrownBy( + () -> + ImmutableFieldLabels.builder().fieldId(0).labels(ImmutableMap.of("k", "v")).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid field id: must be >= 1"); + } + + @Test + public void invalidFieldIdFromJson() { + assertThatThrownBy( + () -> FieldLabelsParser.fromJson("{\"field-id\": 0, \"labels\": {\"k\": \"v\"}}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid field id: must be >= 1"); + } + + @Test + public void emptyLabels() { + assertThatThrownBy(() -> ImmutableFieldLabels.builder().fieldId(1).build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid labels: must be non-empty"); + } + + @Test + public void emptyLabelsFromJson() { + assertThatThrownBy(() -> FieldLabelsParser.fromJson("{\"field-id\": 1, \"labels\": {}}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid labels: must be non-empty"); + } + + @Test + public void missingLabelsFromJson() { + assertThatThrownBy(() -> FieldLabelsParser.fromJson("{\"field-id\": 1}")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse missing map: labels"); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/labels/TestLabelsParser.java b/core/src/test/java/org/apache/iceberg/rest/labels/TestLabelsParser.java new file mode 100644 index 000000000000..c6f1e128aa6f --- /dev/null +++ b/core/src/test/java/org/apache/iceberg/rest/labels/TestLabelsParser.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.rest.labels; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.fasterxml.jackson.databind.JsonNode; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Test; + +public class TestLabelsParser { + + @Test + public void nullCheck() { + assertThatThrownBy(() -> LabelsParser.toJson(null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid labels: null"); + + assertThatThrownBy(() -> LabelsParser.fromJson((JsonNode) null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Cannot parse labels from null object"); + } + + @Test + public void emptyLabels() { + Labels labels = ImmutableLabels.builder().build(); + assertThat(labels.isEmpty()).isTrue(); + + String expectedJson = "{ }"; + assertThat(LabelsParser.toJson(labels, true)).isEqualTo(expectedJson); + + Labels roundTrip = LabelsParser.fromJson(expectedJson); + assertThat(roundTrip.object()).isEmpty(); + assertThat(roundTrip.fields()).isEmpty(); + + Labels nonEmpty = ImmutableLabels.builder().object(ImmutableMap.of("k", "v")).build(); + assertThat(nonEmpty.isEmpty()).isFalse(); + } + + @Test + public void objectLabelsOnly() { + Labels labels = + ImmutableLabels.builder() + .object(ImmutableMap.of("owner", "team-a", "sensitivity", "high")) + .build(); + + String expectedJson = + """ + { + "object" : { + "owner" : "team-a", + "sensitivity" : "high" + } + }"""; + + assertThat(LabelsParser.toJson(labels, true)).isEqualTo(expectedJson); + assertThat(LabelsParser.fromJson(expectedJson)).isEqualTo(labels); + } + + @Test + public void fieldLabelsOnly() { + Labels labels = + ImmutableLabels.builder() + .addFields( + ImmutableFieldLabels.builder() + .fieldId(3) + .labels(ImmutableMap.of("pii", "true")) + .build()) + .build(); + + String expectedJson = + """ + { + "fields" : [ { + "field-id" : 3, + "labels" : { + "pii" : "true" + } + } ] + }"""; + + assertThat(LabelsParser.toJson(labels, true)).isEqualTo(expectedJson); + assertThat(LabelsParser.fromJson(expectedJson)).isEqualTo(labels); + } + + @Test + public void objectAndFieldLabels() { + Labels labels = + ImmutableLabels.builder() + .object(ImmutableMap.of("owner", "team-a")) + .addFields( + ImmutableFieldLabels.builder() + .fieldId(1) + .labels(ImmutableMap.of("classification", "pii")) + .build()) + .addFields( + ImmutableFieldLabels.builder() + .fieldId(2) + .labels(ImmutableMap.of("classification", "public")) + .build()) + .build(); + + String expectedJson = + """ + { + "object" : { + "owner" : "team-a" + }, + "fields" : [ { + "field-id" : 1, + "labels" : { + "classification" : "pii" + } + }, { + "field-id" : 2, + "labels" : { + "classification" : "public" + } + } ] + }"""; + + assertThat(LabelsParser.toJson(labels, true)).isEqualTo(expectedJson); + assertThat(LabelsParser.fromJson(expectedJson)).isEqualTo(labels); + } +} diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponse.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponse.java index 96854fe64c08..a52fa75b37a7 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponse.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponse.java @@ -37,6 +37,9 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.rest.RequestResponseTestBase; +import org.apache.iceberg.rest.labels.ImmutableFieldLabels; +import org.apache.iceberg.rest.labels.ImmutableLabels; +import org.apache.iceberg.rest.labels.Labels; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; @@ -162,6 +165,48 @@ public void testRoundTripSerdeWithV3TableMetadata() throws Exception { assertRoundTripSerializesEquallyFrom(json, resp); } + @Test + public void testRoundTripSerdeWithLabels() throws Exception { + String tableMetadataJson = readTableMetadataInputFile("TableMetadataV2Valid.json"); + TableMetadata metadata = + TableMetadataParser.fromJson(TEST_METADATA_LOCATION, tableMetadataJson); + Labels labels = + ImmutableLabels.builder() + .object(ImmutableMap.of("owner", "team-a")) + .addFields( + ImmutableFieldLabels.builder() + .fieldId(1) + .labels(ImmutableMap.of("classification", "pii")) + .build()) + .build(); + String json = + String.format( + """ + { + "metadata-location":"%s", + "metadata":%s, + "config":{"foo":"bar"}, + "labels":{ + "object":{"owner":"team-a"}, + "fields":[{"field-id":1,"labels":{"classification":"pii"}}] + } + }""", + TEST_METADATA_LOCATION, TableMetadataParser.toJson(metadata)); + + // exercise the full RESTObjectMapper (Jackson) path, not just the parser + LoadTableResponse actual = deserialize(json); + assertThat(actual.labels()).isEqualTo(labels); + + LoadTableResponse resp = + LoadTableResponse.builder() + .withTableMetadata(metadata) + .addAllConfig(CONFIG) + .withLabels(labels) + .build(); + // compare as JSON trees so the readable (pretty) input above is whitespace-insensitive + assertThat(mapper().readTree(serialize(resp))).isEqualTo(mapper().readTree(json)); + } + @Test public void testCanDeserializeWithoutDefaultValues() throws Exception { String metadataJson = readTableMetadataInputFile("TableMetadataV1Valid.json"); @@ -187,6 +232,7 @@ public void assertEquals(LoadTableResponse actual, LoadTableResponse expected) { assertThat(actual.metadataLocation()) .as("Should have the same metadata location") .isEqualTo(expected.metadataLocation()); + assertThat(actual.labels()).as("Should have the same labels").isEqualTo(expected.labels()); } private void assertEqualTableMetadata(TableMetadata actual, TableMetadata expected) { diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponseParser.java index d0a2be263e23..8bc4d82cc2d7 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadTableResponseParser.java @@ -29,6 +29,8 @@ import org.apache.iceberg.TableMetadata; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.rest.credentials.ImmutableCredential; +import org.apache.iceberg.rest.labels.ImmutableFieldLabels; +import org.apache.iceberg.rest.labels.ImmutableLabels; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -478,4 +480,119 @@ public void roundTripSerdeWithCredentials() { assertThat(LoadTableResponseParser.toJson(LoadTableResponseParser.fromJson(json), true)) .isEqualTo(expectedJson); } + + @Test + public void roundTripSerdeWithLabels() { + String uuid = "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b"; + TableMetadata metadata = + TableMetadata.buildFromEmpty() + .assignUUID(uuid) + .setLocation("location") + .setCurrentSchema( + new Schema(Types.NestedField.required(1, "x", Types.LongType.get())), 1) + .addPartitionSpec(PartitionSpec.unpartitioned()) + .addSortOrder(SortOrder.unsorted()) + .discardChanges() + .withMetadataLocation("metadata-location") + .build(); + + LoadTableResponse response = + LoadTableResponse.builder() + .withTableMetadata(metadata) + .withLabels( + ImmutableLabels.builder() + .object(ImmutableMap.of("owner", "team-a")) + .addFields( + ImmutableFieldLabels.builder() + .fieldId(1) + .labels(ImmutableMap.of("classification", "pii")) + .build()) + .build()) + .build(); + + String expectedJson = + String.format( + """ + { + "metadata-location" : "metadata-location", + "metadata" : { + "format-version" : 2, + "table-uuid" : "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b", + "location" : "location", + "last-sequence-number" : 0, + "last-updated-ms" : %d, + "last-column-id" : 1, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "x", + "required" : true, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { }, + "current-snapshot-id" : -1, + "refs" : { }, + "snapshots" : [ ], + "statistics" : [ ], + "partition-statistics" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ ] + }, + "labels" : { + "object" : { + "owner" : "team-a" + }, + "fields" : [ { + "field-id" : 1, + "labels" : { + "classification" : "pii" + } + } ] + } + }""", + metadata.lastUpdatedMillis()); + + String json = LoadTableResponseParser.toJson(response, true); + assertThat(json).isEqualTo(expectedJson); + // can't do an equality comparison because Schema doesn't implement equals/hashCode + assertThat(LoadTableResponseParser.toJson(LoadTableResponseParser.fromJson(json), true)) + .isEqualTo(expectedJson); + } + + @Test + public void labelsAreOptional() { + String uuid = "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b"; + TableMetadata metadata = + TableMetadata.buildFromEmpty() + .assignUUID(uuid) + .setLocation("location") + .setCurrentSchema( + new Schema(Types.NestedField.required(1, "x", Types.LongType.get())), 1) + .addPartitionSpec(PartitionSpec.unpartitioned()) + .addSortOrder(SortOrder.unsorted()) + .discardChanges() + .withMetadataLocation("metadata-location") + .build(); + + LoadTableResponse response = LoadTableResponse.builder().withTableMetadata(metadata).build(); + + String json = LoadTableResponseParser.toJson(response, true); + assertThat(json).doesNotContain("labels"); + assertThat(LoadTableResponseParser.fromJson(json).labels().isEmpty()).isTrue(); + } } diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadViewResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadViewResponseParser.java index f3de08cd2912..63eb9632e2a3 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadViewResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestLoadViewResponseParser.java @@ -25,6 +25,8 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.rest.labels.ImmutableFieldLabels; +import org.apache.iceberg.rest.labels.ImmutableLabels; import org.apache.iceberg.types.Types; import org.apache.iceberg.view.ImmutableViewVersion; import org.apache.iceberg.view.ViewMetadata; @@ -245,4 +247,118 @@ public void roundTripSerdeWithConfig() { assertThat(LoadViewResponseParser.toJson(LoadViewResponseParser.fromJson(json), true)) .isEqualTo(expectedJson); } + + @Test + public void roundTripSerdeWithLabels() { + String uuid = "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b"; + ViewMetadata viewMetadata = + ViewMetadata.builder() + .assignUUID(uuid) + .setLocation("location") + .addSchema(new Schema(Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(0) + .versionId(1) + .timestampMillis(23L) + .defaultNamespace(Namespace.of("ns1")) + .build()) + .setCurrentVersionId(1) + .build(); + + LoadViewResponse response = + ImmutableLoadViewResponse.builder() + .metadata(viewMetadata) + .metadataLocation("custom-location") + .labels( + ImmutableLabels.builder() + .object(ImmutableMap.of("owner", "team-a")) + .addFields( + ImmutableFieldLabels.builder() + .fieldId(1) + .labels(ImmutableMap.of("classification", "pii")) + .build()) + .build()) + .build(); + + String expectedJson = + """ + { + "metadata-location" : "custom-location", + "metadata" : { + "view-uuid" : "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b", + "format-version" : 1, + "location" : "location", + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "x", + "required" : true, + "type" : "long" + } ] + } ], + "current-version-id" : 1, + "versions" : [ { + "version-id" : 1, + "timestamp-ms" : 23, + "schema-id" : 0, + "summary" : { }, + "default-namespace" : [ "ns1" ], + "representations" : [ ] + } ], + "version-log" : [ { + "timestamp-ms" : 23, + "version-id" : 1 + } ] + }, + "labels" : { + "object" : { + "owner" : "team-a" + }, + "fields" : [ { + "field-id" : 1, + "labels" : { + "classification" : "pii" + } + } ] + } + }"""; + + String json = LoadViewResponseParser.toJson(response, true); + assertThat(json).isEqualTo(expectedJson); + // can't do an equality comparison because Schema doesn't implement equals/hashCode + assertThat(LoadViewResponseParser.toJson(LoadViewResponseParser.fromJson(json), true)) + .isEqualTo(expectedJson); + } + + @Test + public void labelsAreOptional() { + String uuid = "386b9f01-002b-4d8c-b77f-42c3fd3b7c9b"; + ViewMetadata viewMetadata = + ViewMetadata.builder() + .assignUUID(uuid) + .setLocation("location") + .addSchema(new Schema(Types.NestedField.required(1, "x", Types.LongType.get()))) + .addVersion( + ImmutableViewVersion.builder() + .schemaId(0) + .versionId(1) + .timestampMillis(23L) + .defaultNamespace(Namespace.of("ns1")) + .build()) + .setCurrentVersionId(1) + .build(); + + LoadViewResponse response = + ImmutableLoadViewResponse.builder() + .metadata(viewMetadata) + .metadataLocation("custom-location") + .build(); + + String json = LoadViewResponseParser.toJson(response, true); + assertThat(json).doesNotContain("labels"); + assertThat(LoadViewResponseParser.fromJson(json).labels().isEmpty()).isTrue(); + } } diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseSparkTable.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseSparkTable.java index 30afa9b7b172..2648f8a759c0 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseSparkTable.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/source/BaseSparkTable.java @@ -29,12 +29,15 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SupportsLabels; import org.apache.iceberg.Table; import org.apache.iceberg.TableOperations; import org.apache.iceberg.TableUtil; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet; import org.apache.iceberg.relocated.com.google.common.collect.Lists; +import org.apache.iceberg.rest.labels.FieldLabels; +import org.apache.iceberg.rest.labels.Labels; import org.apache.iceberg.spark.Spark3Util; import org.apache.iceberg.spark.SparkSchemaUtil; import org.apache.spark.sql.SparkSession; @@ -53,6 +56,8 @@ abstract class BaseSparkTable private static final String LOCATION = "location"; private static final String SORT_ORDER = "sort-order"; private static final String IDENTIFIER_FIELDS = "identifier-fields"; + private static final String LABELS_OBJECT_PREFIX = "labels.object."; + private static final String LABELS_FIELD_PREFIX = "labels.field."; private static final Set RESERVED_PROPERTIES = ImmutableSet.of( PROVIDER, @@ -136,6 +141,21 @@ public Map properties() { .filter(entry -> !RESERVED_PROPERTIES.contains(entry.getKey())) .forEach(propsBuilder::put); + // Surface catalog-provided labels (driver-side only; not part of table state) so they are + // visible in DESCRIBE EXTENDED. The tbl.labels metadata table is the queryable counterpart. + if (table instanceof SupportsLabels) { + Labels labels = ((SupportsLabels) table).labels(); + labels.object().forEach((key, value) -> propsBuilder.put(LABELS_OBJECT_PREFIX + key, value)); + for (FieldLabels fieldLabels : labels.fields()) { + fieldLabels + .labels() + .forEach( + (key, value) -> + propsBuilder.put( + LABELS_FIELD_PREFIX + fieldLabels.fieldId() + "." + key, value)); + } + } + return propsBuilder.build(); }