From 5c09dd1ae2ca4df6504039d014b3f6fdcdfa0a76 Mon Sep 17 00:00:00 2001
From: xodn348 <xodn348@tamu.edu>
Date: Mon, 18 May 2026 07:25:45 +0000
Subject: [PATCH] fix(loading): forward download_config to
 LocalEvaluationModuleFactory for local paths
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previously, both branches of evaluation_module_factory that resolve a local
.py file or directory omitted download_config when constructing
LocalEvaluationModuleFactory. This caused the factory to fall back to a
default DownloadConfig(), silently ignoring caller-supplied settings such as
local_files_only, cache_dir, or use_etag — making it impossible to prevent
outbound network access when loading local evaluation modules.

Fixes #709
---
 src/evaluate/loading.py | 10 ++++++++--
 tests/test_load.py      | 38 +++++++++++++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/src/evaluate/loading.py b/src/evaluate/loading.py
index 505015b15..2049eace9 100644
--- a/src/evaluate/loading.py
+++ b/src/evaluate/loading.py
@@ -617,13 +617,19 @@ def evaluation_module_factory(
     if path.endswith(filename):
         if os.path.isfile(path):
             return LocalEvaluationModuleFactory(
-                path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path
+                path,
+                download_config=download_config,
+                download_mode=download_mode,
+                dynamic_modules_path=dynamic_modules_path,
             ).get_module()
         else:
             raise FileNotFoundError(f"Couldn't find a metric script at {relative_to_absolute_path(path)}")
     elif os.path.isfile(combined_path):
         return LocalEvaluationModuleFactory(
-            combined_path, download_mode=download_mode, dynamic_modules_path=dynamic_modules_path
+            combined_path,
+            download_config=download_config,
+            download_mode=download_mode,
+            dynamic_modules_path=dynamic_modules_path,
         ).get_module()
     elif is_relative_path(path) and path.count("/") <= 1 and not force_local_path:
         try:
diff --git a/tests/test_load.py b/tests/test_load.py
index e20ea671f..dfb3201b8 100644
--- a/tests/test_load.py
+++ b/tests/test_load.py
@@ -1,7 +1,7 @@
 import importlib
 import os
 import tempfile
-from unittest import TestCase
+from unittest import TestCase, mock
 
 import pytest
 from datasets import DownloadConfig
@@ -138,3 +138,39 @@ def test_cache_with_remote_community_module(self):
                 evaluation_module_factory(
                     metric, download_config=self.download_config, dynamic_modules_path=self.dynamic_modules_path
                 )
+
+    def test_evaluation_module_factory_passes_download_config_local_script(self):
+        """Regression test for #709: download_config must be forwarded to LocalEvaluationModuleFactory
+        when loading a local .py file directly."""
+        path = os.path.join(self._metric_loading_script_dir, f"{METRIC_LOADING_SCRIPT_NAME}.py")
+        custom_config = DownloadConfig(cache_dir=self.cache_dir, local_files_only=True)
+        with mock.patch("evaluate.loading.LocalEvaluationModuleFactory") as mock_factory:
+            mock_factory.return_value.get_module.return_value = mock.MagicMock()
+            evaluation_module_factory(
+                path,
+                download_config=custom_config,
+                dynamic_modules_path=self.dynamic_modules_path,
+            )
+            mock_factory.assert_called_once()
+            _, call_kwargs = mock_factory.call_args
+            assert (
+                call_kwargs.get("download_config") is custom_config
+            ), "download_config was not forwarded to LocalEvaluationModuleFactory for .py path"
+
+    def test_evaluation_module_factory_passes_download_config_local_dir(self):
+        """Regression test for #709: download_config must be forwarded to LocalEvaluationModuleFactory
+        when loading a local directory (combined_path case)."""
+        path = self._metric_loading_script_dir
+        custom_config = DownloadConfig(cache_dir=self.cache_dir, local_files_only=True)
+        with mock.patch("evaluate.loading.LocalEvaluationModuleFactory") as mock_factory:
+            mock_factory.return_value.get_module.return_value = mock.MagicMock()
+            evaluation_module_factory(
+                path,
+                download_config=custom_config,
+                dynamic_modules_path=self.dynamic_modules_path,
+            )
+            mock_factory.assert_called_once()
+            _, call_kwargs = mock_factory.call_args
+            assert (
+                call_kwargs.get("download_config") is custom_config
+            ), "download_config was not forwarded to LocalEvaluationModuleFactory for directory path"