From 7daef2f7645115bba74b0b162f7cf78f0962adcf Mon Sep 17 00:00:00 2001 From: Amogh Jahagirdar Date: Thu, 25 Jul 2024 11:45:49 -0600 Subject: [PATCH 1/3] API: Define RepairManifests action interface Co-authored-by: Mathew Fournier <160646114+tabmatfournier@users.noreply.github.com> (cherry picked from commit eb2041b604ffc1d5e0f8793d9c1d4909170efde7) --- .../iceberg/actions/ActionsProvider.java | 6 ++ .../iceberg/actions/RepairManifests.java | 62 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 api/src/main/java/org/apache/iceberg/actions/RepairManifests.java diff --git a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java index 240e34113721..aee1d1ba5a55 100644 --- a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java +++ b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java @@ -94,4 +94,10 @@ default RemoveDanglingDeleteFiles removeDanglingDeleteFiles(Table table) { throw new UnsupportedOperationException( this.getClass().getName() + " does not implement removeDanglingDeleteFiles"); } + + /** Instantiates an action to repair manifests */ + default RepairManifests repairManifests(Table table) { + throw new UnsupportedOperationException( + this.getClass().getName() + " does not implement repairManifests"); + } } diff --git a/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java b/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java new file mode 100644 index 000000000000..cb7db0c54241 --- /dev/null +++ b/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.actions; + +import org.apache.iceberg.ManifestFile; + +/** An action that will repair manifests. Implementations should produce a new set of manifests. */ +public interface RepairManifests extends SnapshotUpdate { + + /** Configuration method for repairing manifest entry statistics */ + RepairManifests repairEntryStats(); + + /** + * Configuration method for removing duplicate file entries and removing files which no longer + * exist in storage + */ + RepairManifests repairFileEntries(); + + /** + * Configuration option for determining the rewritten and added manifests without actually + * committing the operation to the table + * + * @return this for method chaining + */ + RepairManifests dryRun(); + + interface Result { + /** Returns rewritten manifests. */ + Iterable rewrittenManifests(); + + /** Returns the duplicate file paths removed */ + Iterable duplicateFilesRemoved(); + + /** Returns the paths of the missing files which were removed */ + Iterable missingFilesRemoved(); + + /** Returns the paths of the missing files which were recovered */ + Iterable missingFilesRecovered(); + + /** Returns the number of manifest entries for which stats were incorrect */ + long entryStatsIncorrectCount(); + + /** Returns the number of manifest entries for which stats were corrected */ + long entryStatsRepairedCount(); + } +} From 14c53b4c1d4334f90d7bb16ec9907a46080150ef Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Tue, 28 Jul 2026 19:04:56 +0000 Subject: [PATCH 2/3] API: Scope RepairManifests to entry-stats repair Narrows the RepairManifests action interface to its first-cut scope: repairing incorrect manifest entry statistics, with a dry-run option. - Drop repairFileEntries() and the duplicate/missing-file Result methods. Removing duplicate entries and dropping entries for missing files changes the logical contents of the table (rows deleted or, for delete files, resurrected) and warrants its own design; it can be added back as a follow-up once those semantics are settled. - Repairing entry stats leaves table contents unchanged and preserves the live file count, so it is a self-contained first step. - Tighten the javadoc: describe what each method does and add @return tags and the missing periods. --- .../iceberg/actions/ActionsProvider.java | 2 +- .../iceberg/actions/RepairManifests.java | 36 +++++++++---------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java index aee1d1ba5a55..55e412708312 100644 --- a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java +++ b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java @@ -95,7 +95,7 @@ default RemoveDanglingDeleteFiles removeDanglingDeleteFiles(Table table) { this.getClass().getName() + " does not implement removeDanglingDeleteFiles"); } - /** Instantiates an action to repair manifests */ + /** Instantiates an action to repair the manifests of a table. */ default RepairManifests repairManifests(Table table) { throw new UnsupportedOperationException( this.getClass().getName() + " does not implement repairManifests"); diff --git a/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java b/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java index cb7db0c54241..7022c092a77b 100644 --- a/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java +++ b/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java @@ -20,43 +20,39 @@ import org.apache.iceberg.ManifestFile; -/** An action that will repair manifests. Implementations should produce a new set of manifests. */ +/** + * An action that repairs incorrect statistics in the manifests of a table. + * + *

Implementations rewrite the manifests of the table, producing a new set of manifests in which + * the statistics of the entries are corrected to match the underlying data and delete files. + */ public interface RepairManifests extends SnapshotUpdate { - /** Configuration method for repairing manifest entry statistics */ - RepairManifests repairEntryStats(); - /** - * Configuration method for removing duplicate file entries and removing files which no longer - * exist in storage + * Repairs incorrect statistics of manifest entries, such as record counts, file sizes and column + * level statistics. + * + * @return this for method chaining */ - RepairManifests repairFileEntries(); + RepairManifests repairEntryStats(); /** - * Configuration option for determining the rewritten and added manifests without actually - * committing the operation to the table + * Determines the repairs that would be performed without actually committing the operation to the + * table. * * @return this for method chaining */ RepairManifests dryRun(); + /** The action result that contains a summary of the execution. */ interface Result { /** Returns rewritten manifests. */ Iterable rewrittenManifests(); - /** Returns the duplicate file paths removed */ - Iterable duplicateFilesRemoved(); - - /** Returns the paths of the missing files which were removed */ - Iterable missingFilesRemoved(); - - /** Returns the paths of the missing files which were recovered */ - Iterable missingFilesRecovered(); - - /** Returns the number of manifest entries for which stats were incorrect */ + /** Returns the number of manifest entries for which stats were incorrect. */ long entryStatsIncorrectCount(); - /** Returns the number of manifest entries for which stats were corrected */ + /** Returns the number of manifest entries for which stats were corrected. */ long entryStatsRepairedCount(); } } From 4ab692ba6f4b3d4c66aad51487970f31c6de6c7d Mon Sep 17 00:00:00 2001 From: rahulsmahadev Date: Fri, 31 Jul 2026 00:53:57 +0000 Subject: [PATCH 3/3] API: Address review comments on RepairManifests result naming - Rename Result#rewrittenManifests to repairedManifests, so the result uses "repaired" consistently with the rest of the action. - Rename entryStatsIncorrectCount/entryStatsRepairedCount to incorrectEntryCount/repairedEntryCount. --- .../java/org/apache/iceberg/actions/RepairManifests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java b/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java index 7022c092a77b..38aedb239a3f 100644 --- a/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java +++ b/api/src/main/java/org/apache/iceberg/actions/RepairManifests.java @@ -46,13 +46,13 @@ public interface RepairManifests extends SnapshotUpdate rewrittenManifests(); + /** Returns the repaired manifests. */ + Iterable repairedManifests(); /** Returns the number of manifest entries for which stats were incorrect. */ - long entryStatsIncorrectCount(); + long incorrectEntryCount(); /** Returns the number of manifest entries for which stats were corrected. */ - long entryStatsRepairedCount(); + long repairedEntryCount(); } }