From eae81c56dfde9f440b9a1dcbd0fa53b8631279e0 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 20:44:33 +0000 Subject: [PATCH 1/6] Use feature flag to disable Android v4 API --- .../_internal/base/feature_flags.py | 1 + .../platforms/android/fetch_artifact.py | 19 +++++++++++++++++++ .../_internal/platforms/android/flash.py | 3 +++ .../platforms/android/symbols_downloader.py | 11 ++++++++++- 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/base/feature_flags.py b/src/clusterfuzz/_internal/base/feature_flags.py index 5dfeccae1c1..9660745990c 100644 --- a/src/clusterfuzz/_internal/base/feature_flags.py +++ b/src/clusterfuzz/_internal/base/feature_flags.py @@ -44,6 +44,7 @@ class FeatureFlags(Enum): ENABLE_FUZZ_FOR_BOTS = 'enable_fuzz_for_bots' STORAGE_THREADED_OPS_FUZZ_TARGETS = 'storage_threaded_ops_fuzz_targets' + CALL_ANDROID_API = 'call_android_api' @property def flag(self): diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index e86cced06c6..6a2eb35b707 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -25,6 +25,7 @@ import apiclient from oauth2client.service_account import ServiceAccountCredentials +from clusterfuzz._internal.base import feature_flags from clusterfuzz._internal.config import db_config from clusterfuzz._internal.google_cloud_utils import storage from clusterfuzz._internal.metrics import logs @@ -64,6 +65,14 @@ def _use_v4(): return False +def _call_android_api_enabled(): + """Return True if we should call the Android Build API. Enabled by default.""" + flag = feature_flags.FeatureFlags.CALL_ANDROID_API.flag + if flag is None: + return True + return flag.enabled + + def execute_request_with_retries(request): """Executes request and retries on failure.""" result = None @@ -332,6 +341,11 @@ def get_stable_build_info(): def get_latest_artifact_info(branch, target, signed=False, stable_build=False): """Return latest artifact for a branch and target.""" + if not _call_android_api_enabled(): + logs.warning( + 'Android build API is disabled by feature flag call_android_api.') + return None + client = get_client() if not client: return None @@ -405,6 +419,11 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): def get(bid, target, regex, output_directory, output_filename=None): """Return artifact for a given build id, target and file regex.""" + if not _call_android_api_enabled(): + logs.warning( + 'Android build API is disabled by feature flag call_android_api.') + return None + client = get_client() if not client: return None diff --git a/src/clusterfuzz/_internal/platforms/android/flash.py b/src/clusterfuzz/_internal/platforms/android/flash.py index b292b1491e4..83e689e0ba5 100644 --- a/src/clusterfuzz/_internal/platforms/android/flash.py +++ b/src/clusterfuzz/_internal/platforms/android/flash.py @@ -97,6 +97,9 @@ def boot_stable_build_cuttlefish(branch, target, image_directory): """Boot cuttlefish instance using stable build id fetched from gcs.""" build_info = fetch_artifact.get_latest_artifact_info( branch, target, stable_build=True) + if not build_info: + logs.error('Unable to fetch stable build info for cuttlefish.') + return download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) adb.recreate_cuttlefish_device() adb.connect_to_cuttlefish_device() diff --git a/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py b/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py index 112d86fc215..55a906b5498 100644 --- a/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py +++ b/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py @@ -196,7 +196,12 @@ def download_trusty_symbols_if_needed(symbols_directory, app_name, bid): branch = 'polygon-trusty-whitechapel-master' if not bid: - bid = fetch_artifact.get_latest_artifact_info(branch, ab_target)['bid'] + build_info = fetch_artifact.get_latest_artifact_info(branch, ab_target) + if not build_info: + logs.error(f'Unable to fetch build info for branch {branch} ' + f'and target {ab_target}.') + return + bid = build_info['bid'] artifact_filename = f'{ab_target}-{bid}.syms.zip' symbols_archive_path = os.path.join(symbols_directory, artifact_filename) @@ -204,6 +209,10 @@ def download_trusty_symbols_if_needed(symbols_directory, app_name, bid): download_artifact_if_needed(bid, symbols_directory, symbols_archive_path, [ab_target], artifact_filename, None) + if not os.path.exists(symbols_archive_path): + logs.error(f'Unable to locate symbols archive {symbols_archive_path}.') + return + with zipfile.ZipFile(symbols_archive_path, 'r') as symbols_zipfile: for filepath in symbols_zipfile.namelist(): if f'{app_name}.syms.elf' in filepath: From 629a1a6489fd36e889cf218d2a71225d0a40a243 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 23:01:47 +0000 Subject: [PATCH 2/6] [Android] Refactor artifact fetching and flash logic to prevent bot loops When the Android Build API feature flag is disabled, or when artifacts are missing, bots can enter a cycling state. Previously, `fetch_artifact.py` explicitly called `adb.bad_state_reached()`. For uworkers, this forced an un-trapped `sys.exit(-1)` that caused them to silently abandon tasks and cycle. This commit delegatess the error handling of `fetch_artifact` failures to its callers. We also updated `flash.py` to only trigger `adb.bad_state_reached()` on physical devices if the API is disabled and no local images are present, forcing the host to wait 1 hour rather than spinning. Guarded `download_latest_build` checks allow devices with local images to reuse them when the API is toggled off.TAG=agyCONV=773a0adb-dc53-4706-b185-52bf9391df88 --- .../platforms/android/fetch_artifact.py | 3 --- .../_internal/platforms/android/flash.py | 26 +++++++++++++------ 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index 6a2eb35b707..d3e4c68e8dd 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -31,8 +31,6 @@ from clusterfuzz._internal.metrics import logs from clusterfuzz._internal.system import environment -from . import adb - # 20 MB default chunk size. DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 @@ -281,7 +279,6 @@ def get_artifacts_for_build(client, if not artifacts: logs.error(f'No artifact found for target {target}, build id {bid}.\n' f'request {request_str}, results {results}') - adb.bad_state_reached() return artifacts diff --git a/src/clusterfuzz/_internal/platforms/android/flash.py b/src/clusterfuzz/_internal/platforms/android/flash.py index 83e689e0ba5..020762a8a74 100644 --- a/src/clusterfuzz/_internal/platforms/android/flash.py +++ b/src/clusterfuzz/_internal/platforms/android/flash.py @@ -85,7 +85,7 @@ def download_latest_build(build_info, image_regexes, image_directory): logs.error('Failed to download artifact %s for ' 'branch %s and target %s.' % (image_file_paths, build_info['branch'], target)) - return + adb.bad_state_reached() for file_path in image_file_paths: if file_path.endswith('.zip') or file_path.endswith('.tar.gz'): @@ -99,8 +99,8 @@ def boot_stable_build_cuttlefish(branch, target, image_directory): branch, target, stable_build=True) if not build_info: logs.error('Unable to fetch stable build info for cuttlefish.') - return - download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) + else: + download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) adb.recreate_cuttlefish_device() adb.connect_to_cuttlefish_device() @@ -182,12 +182,21 @@ def flash_to_latest_build_if_needed(): if not build_info: logs.error('Unable to fetch information on latest build artifact for ' 'branch %s and target %s.' % (branch, target)) - return + + has_local_images = os.path.exists(image_directory) and bool( + os.listdir(image_directory)) + if not build_info and not has_local_images: + logs.error( + 'No valid build info available (API may be disabled) and no local ' + 'images found in %s. Bot cannot recover.' % image_directory) + adb.bad_state_reached() instance_id = None is_candidate = None if environment.is_android_cuttlefish(): - download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) + if build_info: + download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, + image_directory) adb.recreate_cuttlefish_device() adb.connect_to_cuttlefish_device() if compute_metadata.is_gce(): @@ -196,7 +205,8 @@ def flash_to_latest_build_if_needed(): # Determine if the current instance is a candidate. is_candidate = utils.get_clusterfuzz_release() == 'candidate' else: - download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory) + if build_info: + download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory) # We do one device flash at a time on one host, otherwise we run into # failures and device being stuck in a bad state. flash_lock_key_name = 'flash:%s' % socket.gethostname() @@ -239,7 +249,7 @@ def flash_to_latest_build_if_needed(): if environment.is_android_cuttlefish(): logs.info('Trying to boot cuttlefish instance using stable build.') monitoring_metrics.CF_TIP_BOOT_FAILED_COUNT.increment({ - 'build_id': build_info['bid'], + 'build_id': build_info['bid'] if build_info else 'unknown', 'instance_id': instance_id, 'is_candidate': is_candidate, 'is_succeeded': False @@ -252,7 +262,7 @@ def flash_to_latest_build_if_needed(): logs.error('Unable to find device. Reimaging failed.') adb.bad_state_reached() monitoring_metrics.CF_TIP_BOOT_FAILED_COUNT.increment({ - 'build_id': build_info['bid'], + 'build_id': build_info['bid'] if build_info else 'unknown', 'instance_id': instance_id, 'is_candidate': is_candidate, 'is_succeeded': True From 3f568fa1b61a5233d84fc2c244777f7f08ea8a3e Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 23:13:38 +0000 Subject: [PATCH 3/6] Fixes linter errors --- src/clusterfuzz/_internal/platforms/android/fetch_artifact.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index a2c195756b4..152d357a064 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -288,7 +288,7 @@ def _get_artifacts_for_build(client, if not artifacts: logs.error(f'No artifact found for target {target}, build id {bid}.\n' - f'request {request_str}, results {results}') + f'results {results}') return artifacts From 9d8822a4e7e47b25ef18e312eccffaa0996a8b04 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 23:35:50 +0000 Subject: [PATCH 4/6] Add feature flag mocking to fetch_artifact_test --- src/clusterfuzz/_internal/platforms/android/flash.py | 2 +- .../tests/core/platforms/android/fetch_artifact_test.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/platforms/android/flash.py b/src/clusterfuzz/_internal/platforms/android/flash.py index 020762a8a74..6b58c895c93 100644 --- a/src/clusterfuzz/_internal/platforms/android/flash.py +++ b/src/clusterfuzz/_internal/platforms/android/flash.py @@ -188,7 +188,7 @@ def flash_to_latest_build_if_needed(): if not build_info and not has_local_images: logs.error( 'No valid build info available (API may be disabled) and no local ' - 'images found in %s. Bot cannot recover.' % image_directory) + f'images found in {image_directory}. Bot cannot recover.') adb.bad_state_reached() instance_id = None diff --git a/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py b/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py index e01de1ba822..6d7b42b88d4 100644 --- a/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py +++ b/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py @@ -29,10 +29,12 @@ def setUp(self): helpers.patch(self, [ 'clusterfuzz._internal.platforms.android.fetch_artifact._get_client', 'clusterfuzz._internal.platforms.android.fetch_artifact._use_v4', + 'clusterfuzz._internal.platforms.android.fetch_artifact._call_android_api_enabled', 'clusterfuzz._internal.platforms.android.fetch_artifact._execute_request_with_retries', ]) self.mock_client = mock.MagicMock() self.mock._get_client.return_value = self.mock_client + self.mock._call_android_api_enabled.return_value = True def test_get_latest_artifact_v4_success(self): """Tests get_latest_artifact_info (V4). Expects extraction of {bid, branch, target} when list_builds returns data.""" @@ -116,6 +118,13 @@ def test_get_latest_artifact_client_auth_failure(self): result = fetch_artifact.get_latest_artifact_info('branch1', 'target1') self.assertIsNone(result) + def test_get_latest_artifact_disabled_by_feature_flag(self): + """Tests get_latest_artifact_info exits early and returns None when disabled by feature flag.""" + self.mock._call_android_api_enabled.return_value = False + + result = fetch_artifact.get_latest_artifact_info('branch1', 'target1') + self.assertIsNone(result) + def test_get_artifacts_for_build_empty_regexp(self): """Tests _get_artifacts_for_build returns [] returning early when regexp is empty, bypassing API calls.""" result = fetch_artifact._get_artifacts_for_build( From 70c3ef7f61bf693923f4b5fdb90951c5b5ef63f0 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Wed, 22 Jul 2026 20:55:17 +0000 Subject: [PATCH 5/6] Restore early return in flash.py if build_info missing --- .../_internal/platforms/android/flash.py | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/clusterfuzz/_internal/platforms/android/flash.py b/src/clusterfuzz/_internal/platforms/android/flash.py index 6b58c895c93..283a80120d3 100644 --- a/src/clusterfuzz/_internal/platforms/android/flash.py +++ b/src/clusterfuzz/_internal/platforms/android/flash.py @@ -182,21 +182,12 @@ def flash_to_latest_build_if_needed(): if not build_info: logs.error('Unable to fetch information on latest build artifact for ' 'branch %s and target %s.' % (branch, target)) - - has_local_images = os.path.exists(image_directory) and bool( - os.listdir(image_directory)) - if not build_info and not has_local_images: - logs.error( - 'No valid build info available (API may be disabled) and no local ' - f'images found in {image_directory}. Bot cannot recover.') - adb.bad_state_reached() + return instance_id = None is_candidate = None if environment.is_android_cuttlefish(): - if build_info: - download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, - image_directory) + download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) adb.recreate_cuttlefish_device() adb.connect_to_cuttlefish_device() if compute_metadata.is_gce(): @@ -205,8 +196,7 @@ def flash_to_latest_build_if_needed(): # Determine if the current instance is a candidate. is_candidate = utils.get_clusterfuzz_release() == 'candidate' else: - if build_info: - download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory) + download_latest_build(build_info, FLASH_IMAGE_REGEXES, image_directory) # We do one device flash at a time on one host, otherwise we run into # failures and device being stuck in a bad state. flash_lock_key_name = 'flash:%s' % socket.gethostname() @@ -249,7 +239,7 @@ def flash_to_latest_build_if_needed(): if environment.is_android_cuttlefish(): logs.info('Trying to boot cuttlefish instance using stable build.') monitoring_metrics.CF_TIP_BOOT_FAILED_COUNT.increment({ - 'build_id': build_info['bid'] if build_info else 'unknown', + 'build_id': build_info['bid'], 'instance_id': instance_id, 'is_candidate': is_candidate, 'is_succeeded': False @@ -262,7 +252,7 @@ def flash_to_latest_build_if_needed(): logs.error('Unable to find device. Reimaging failed.') adb.bad_state_reached() monitoring_metrics.CF_TIP_BOOT_FAILED_COUNT.increment({ - 'build_id': build_info['bid'] if build_info else 'unknown', + 'build_id': build_info['bid'], 'instance_id': instance_id, 'is_candidate': is_candidate, 'is_succeeded': True From 46b5a605b7bc80f99aceba87ba74f792a7a8d071 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Wed, 22 Jul 2026 23:58:16 +0000 Subject: [PATCH 6/6] Uworker can't read the feature_flag --- .../platforms/android/fetch_artifact.py | 8 +++- .../platforms/android/fetch_artifact_test.py | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index 152d357a064..961e012bc50 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -72,7 +72,13 @@ def _use_v4(): def _call_android_api_enabled(): - """Return True if we should call the Android Build API. Enabled by default.""" + """Return True if we should call the Android Build API, enabled by default, + Disabled always if invoked in a uworker + """ + if environment.is_uworker(): + logs.info('AndroidBuildAPI access disabled for uworker.') + return False + flag = feature_flags.FeatureFlags.CALL_ANDROID_API.flag if flag is None: return True diff --git a/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py b/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py index 6d7b42b88d4..5417a64e229 100644 --- a/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py +++ b/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py @@ -18,6 +18,7 @@ import unittest from unittest import mock +from clusterfuzz._internal.base import feature_flags from clusterfuzz._internal.platforms.android import fetch_artifact from clusterfuzz._internal.tests.test_libs import helpers @@ -132,3 +133,42 @@ def test_get_artifacts_for_build_empty_regexp(self): self.assertEqual(result, []) self.mock_client.list_artifacts.assert_not_called() self.mock_client.buildartifact().list.assert_not_called() + + +class CallAndroidApiEnabledTest(unittest.TestCase): + """Tests for _call_android_api_enabled.""" + + def setUp(self): + helpers.patch(self, [ + 'clusterfuzz._internal.system.environment.is_uworker', + ]) + self.mock.is_uworker.return_value = False + + def test_is_uworker_returns_false(self): + self.mock.is_uworker.return_value = True + self.assertFalse(fetch_artifact._call_android_api_enabled()) + + def test_flag_none_returns_true(self): + with mock.patch.object( + feature_flags.FeatureFlags, 'flag', + new_callable=mock.PropertyMock) as mock_flag: + mock_flag.return_value = None + self.assertTrue(fetch_artifact._call_android_api_enabled()) + + def test_flag_enabled_returns_true(self): + mock_flag_obj = mock.MagicMock() + mock_flag_obj.enabled = True + with mock.patch.object( + feature_flags.FeatureFlags, 'flag', + new_callable=mock.PropertyMock) as mock_flag: + mock_flag.return_value = mock_flag_obj + self.assertTrue(fetch_artifact._call_android_api_enabled()) + + def test_flag_disabled_returns_false(self): + mock_flag_obj = mock.MagicMock() + mock_flag_obj.enabled = False + with mock.patch.object( + feature_flags.FeatureFlags, 'flag', + new_callable=mock.PropertyMock) as mock_flag: + mock_flag.return_value = mock_flag_obj + self.assertFalse(fetch_artifact._call_android_api_enabled())