[Migration] Migration to Datafusion 54 - Test

.github/actions/setup-submodules/action.yaml

@@ -0,0 +1,15 @@
+name: "Setup Submodules"
+description: "Initialize and update git submodules for testing"
+runs:
+  using: "composite"
+  steps:
+    - name: Initialize and update submodules
+      shell: bash
+      run: |
+        git config --global --add safe.directory $GITHUB_WORKSPACE
+        git submodule init
+        # Override update=none setting for CI
+        git config submodule.testing.update checkout
+        git config submodule.parquet-testing.update checkout
+        git config submodule.datafusion-testing.update checkout
+        git submodule update --depth 1

.github/workflows/dependencies.yml

@@ -43,8 +43,9 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:

.github/workflows/docs_pr.yaml

@@ -44,8 +44,9 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup uv
         uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b  # v8.1.0
       - name: Install doc dependencies

.github/workflows/extended.yml

@@ -57,6 +57,34 @@ permissions:
   checks: write
 
 jobs:
+
+  # Check crate compiles and base cargo check passes
+  linux-build-lib:
+    name: linux build test
+    runs-on: ${{ github.repository_owner == 'apache' && format('runs-on={0},family=m8a,cpu=8,image=ubuntu24-full-x64,extras=s3-cache,disk=large,tag=datafusion', github.run_id) || 'ubuntu-latest' }}
+    # note: do not use amd/rust container to preserve disk space
+    steps:
+      - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e  # v2.0.3
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
+          fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
+      - name: Install Rust
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+          source $HOME/.cargo/env
+          rustup toolchain install
+      - name: Install Protobuf Compiler
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y protobuf-compiler
+      - name: Prepare cargo build
+        run: |
+          cargo check --profile ci --all-targets
+          cargo clean
+
   # Run extended tests (with feature 'extended_tests')
   linux-test-extended:
     name: cargo test 'extended_tests' (amd64)
@@ -67,8 +95,9 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Free Disk Space (Ubuntu)
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be  # v1.3.1
       - name: Install Rust
@@ -114,8 +143,9 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -136,8 +166,9 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           ref: ${{ github.event.inputs.pr_head_sha }} # will be empty if triggered by push
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       # Don't use setup-builder to avoid configuring RUST_BACKTRACE which is expensive
       - name: Install protobuf compiler
         run: |

.github/workflows/rust.yml

@@ -279,8 +279,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -325,8 +326,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         run: rustup toolchain install stable
       - name: Rust Dependency Cache
@@ -357,8 +359,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -388,8 +391,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -451,8 +455,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -499,8 +504,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -524,8 +530,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -567,8 +574,9 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-macos-aarch64-builder
       - name: Run datafusion-ffi tests
@@ -660,8 +668,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -685,8 +694,9 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -707,8 +717,9 @@ jobs:
       - uses: runs-on/action@d141ef83eb66d096ce8afc767e09115a65c63b60  # v2.1.2
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
         with:
@@ -742,8 +753,9 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
-          submodules: true
           fetch-depth: 1
+      - name: Setup Submodules
+        uses: ./.github/actions/setup-submodules
 
       - name: Mark repository as safe for git
         # Required for git commands inside container (avoids "dubious ownership" error)

.gitmodules

@@ -1,10 +1,13 @@
 [submodule "parquet-testing"]
 	path = parquet-testing
 	url = https://github.com/apache/parquet-testing.git
+	update = none
 [submodule "testing"]
 	path = testing
 	url = https://github.com/apache/arrow-testing
+	update = none
 [submodule "datafusion-testing"]
 	path = datafusion-testing
 	url = https://github.com/apache/datafusion-testing.git
 	branch = main
+	update = none

datafusion/datasource/src/source.rs

@@ -123,6 +123,18 @@ use datafusion_physical_plan::filter_pushdown::{
 ///    └─────────────────────┘
 /// ```
 pub trait DataSource: Any + Send + Sync + Debug {
+    /// Optionally delegates downcast identity to an inner [`DataSource`].
+    ///
+    /// Wrapper types (e.g. a partitioning adapter that wraps a [`FileScanConfig`])
+    /// can implement this to make `is` and `downcast_ref` transparently see
+    /// through to the inner source, exactly as [`ExecutionPlan::downcast_delegate`]
+    /// does for execution plans.
+    ///
+    /// Implementations should return the inner source, not `self`.
+    fn downcast_delegate(&self) -> Option<&dyn DataSource> {
+        None
+    }
+
     /// Open the specified output partition and return its stream of
     /// [`RecordBatch`]es.
     ///
@@ -292,11 +304,17 @@ impl OpenArgs {
 
 impl dyn DataSource {
     pub fn is<T: DataSource>(&self) -> bool {
-        (self as &dyn Any).is::<T>()
+        match self.downcast_delegate() {
+            Some(delegate) => delegate.is::<T>(),
+            None => (self as &dyn Any).is::<T>(),
+        }
     }
 
     pub fn downcast_ref<T: DataSource>(&self) -> Option<&T> {
-        (self as &dyn Any).downcast_ref()
+        match self.downcast_delegate() {
+            Some(delegate) => delegate.downcast_ref::<T>(),
+            None => (self as &dyn Any).downcast_ref(),
+        }
     }
 }
 

datafusion/datasource/src/statistics.rs

@@ -97,8 +97,16 @@ impl MinMaxStatistics {
                             .zip(s.column_statistics[i].max_value.get_value().cloned())
                             .ok_or_else(|| plan_datafusion_err!("statistics not found"))
                     } else {
-                        let partition_value = &pv[i - s.column_statistics.len()];
-                        Ok((partition_value.clone(), partition_value.clone()))
+                        if let Some(partition_value) =
+                            pv.get(i - s.column_statistics.len())
+                        {
+                            Ok((partition_value.clone(), partition_value.clone()))
+                        } else {
+                            Err(plan_datafusion_err!(
+                                "statistics not found for partition, expected at most {}",
+                                s.column_statistics.len()
+                            ))
+                        }
                     }
                 })
                 .collect::<Result<Vec<_>>>()?
@@ -890,4 +898,31 @@ mod tests {
 
         Ok(())
     }
+
+    #[test]
+    fn min_max_statistics_missing_column_stats_returns_error() {
+        let schema = test_schema();
+        let sort_order =
+            [PhysicalSortExpr::new_default(Arc::new(Column::new("a", 0)))].into();
+        let files = [
+            file_with_stats("f1.parquet", Statistics::default()),
+            file_with_stats("f2.parquet", Statistics::default()),
+        ];
+
+        let err = match MinMaxStatistics::new_from_files(
+            &sort_order,
+            &schema,
+            None,
+            files.iter(),
+        ) {
+            Ok(_) => panic!("expected missing statistics error"),
+            Err(err) => err,
+        };
+
+        assert!(
+            err.to_string()
+                .contains("statistics not found for partition"),
+            "unexpected error: {err:?}"
+        );
+    }
 }