From b185fc1eae511a96f2b11031648674602f170d6c Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 06:15:31 +0000 Subject: [PATCH 1/6] Add Android Build API V4 REST client and tests --- .../platforms/android/android_build_v4_api.py | 285 ++++++++++++++++++ .../android/android_build_v4_api_test.py | 257 ++++++++++++++++ 2 files changed, 542 insertions(+) create mode 100644 src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py create mode 100644 src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py diff --git a/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py new file mode 100644 index 00000000000..4b46a0ac7e7 --- /dev/null +++ b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py @@ -0,0 +1,285 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""HTTP client for Android Build API V4 REST endpoints.""" + +import os +from urllib.parse import quote + +import google.auth.exceptions +from google.auth.transport.requests import Request +from google.oauth2 import service_account +import requests + +from clusterfuzz._internal.base import retry +from clusterfuzz._internal.metrics import logs + +# HTTP request timeout in seconds. +HTTP_TIMEOUT = 60 + +# 20 MB default chunk size for file downloads. +DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 + + +class AndroidBuildV4Api: + """HTTP client for Android Build API V4 REST endpoints.""" + + BASE_URL = 'https://androidbuild-pa.googleapis.com' + + def __init__(self, token: str): + """Initializes the Android Build API V4 client. + + Do not call this directly. Use AndroidBuildV4Api.create_authenticated() + instead. + + Args: + token: OAuth2 authorization bearer token. + """ + self.token = token + + @staticmethod + def create_authenticated( + credentials: service_account.Credentials) -> 'AndroidBuildV4Api': + """Validates credentials and constructs an authenticated client instance. + + Args: + credentials: Service account credentials for authenticating requests. + + Returns: + An instance of AndroidBuildV4Api ready for requests. + + Raises: + google.auth.exceptions.GoogleAuthError: If authentication token refresh + fails. + """ + if not credentials.valid: + credentials.refresh(Request()) + return AndroidBuildV4Api(credentials.token) + + def _get_headers(self) -> dict[str, str]: + """Returns headers required for Android Build API V4 requests. + + Returns: + A dictionary containing the Authorization and Accept headers. + """ + return { + 'Authorization': f'Bearer {self.token}', + 'Accept': 'application/json', + } + + @retry.wrap( + retries=5, delay=1, backoff=2, function='android_build_v4_api._request') + def _request(self, + url: str, + params: dict | None = None, + headers: dict | None = None, + stream: bool = False) -> requests.Response: + """Launches an HTTP GET request with retries. + + Args: + url: Target URL for the GET request. + params: Optional query parameters dictionary. + headers: Optional additional headers dictionary. + stream: Whether to stream the response content (for file downloads). + + Returns: + The HTTP response object. + + Raises: + requests.exceptions.RequestException: If the HTTP request fails or + returns an error status. + """ + request_headers = self._get_headers() + if headers: + request_headers.update(headers) + + logs.info(f'AndroidBuildV4Api request for {url}', params=params) + response = requests.get( + url, + headers=request_headers, + params=params, + stream=stream, + timeout=HTTP_TIMEOUT) + response.raise_for_status() + return response + + @retry.wrap( + retries=5, + delay=1, + backoff=2, + function='android_build_v4_api._download_file') + def _download_file(self, url: str, output_path: str) -> None: + """Downloads content from a URL to output_path with retries. + + Args: + url: Download URL. + output_path: Local filesystem path where the file should be saved. + + Raises: + requests.exceptions.RequestException: If the HTTP download fails. + OSError: If creating directories or writing the file fails. + """ + dirname = os.path.dirname(output_path) + if dirname: + os.makedirs(dirname, exist_ok=True) + + response = requests.get(url, stream=True, timeout=HTTP_TIMEOUT) + response.raise_for_status() + with open(output_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=DEFAULT_CHUNK_SIZE): + if chunk: + f.write(chunk) + + def list_builds(self, branch: str, target: str, + signed: bool = False) -> dict | None: + """List builds for a branch and target. + + Args: + branch: Android build branch (e.g. 'git_main'). + target: Android build target (e.g. 'cf_x86_64_phone-next-userdebug'). + signed: Whether to request signed builds only. + + Returns: + JSON response dictionary containing builds, or None on failure. + """ + params = { + 'buildType': 'submitted', + 'branches': branch, + 'targets': target, + 'successful': 'true', + 'pageSize': 1, + } + if signed: + params['signed'] = 'true' + + url = f'{self.BASE_URL}/v4/builds' + try: + response = self._request(url, params=params) + return response.json() + except (requests.exceptions.RequestException, + google.auth.exceptions.GoogleAuthError, ValueError) as e: + logs.error( + f'V4 list_builds failed for branch {branch}, target {target}: {e}') + return None + + def list_artifacts(self, + bid: str, + target: str, + attempt_id: str = 'latest', + regexp: str | None = None, + page_size: int = 100) -> list: + """List artifacts for a given build. + + Args: + bid: Android build ID. + target: Android build target name. + attempt_id: Build attempt identifier (defaults to 'latest'). + regexp: Optional regular expression pattern to filter artifact names. + page_size: Number of artifacts per page (defaults to 100). + + Returns: + List of artifact dictionary objects. + """ + params = {'pageSize': page_size} + if regexp: + params['nameRegexp'] = regexp + + path = f'/v4/builds/{bid}/{target}/attempts/{attempt_id}/artifacts' + url = f'{self.BASE_URL}{path}' + artifacts = [] + + page_token = None + while True: + if page_token: + params['pageToken'] = page_token + + try: + response = self._request(url, params=params) + result = response.json() + except (requests.exceptions.RequestException, + google.auth.exceptions.GoogleAuthError, ValueError) as e: + logs.error( + f'V4 list_artifacts failed for build {bid}, target {target}: {e}') + break + + if 'artifacts' in result: + artifacts.extend(result['artifacts']) + + page_token = result.get('nextPageToken') + if not page_token: + break + + return artifacts + + def get_artifact_metadata(self, bid: str, target: str, attempt_id: str, + name: str) -> dict | None: + """Get artifact metadata. + + Args: + bid: Android build ID. + target: Android build target name. + attempt_id: Build attempt identifier. + name: Artifact name. + + Returns: + Artifact metadata dictionary, or None on failure. + """ + resource_id = quote(name, safe='') + path = (f'/v4/builds/{bid}/{target}/attempts/{attempt_id}' + f'/artifacts/{resource_id}') + url = f'{self.BASE_URL}{path}' + try: + response = self._request(url) + data = response.json() + return data.get('buildArtifactMetadata', data) + except (requests.exceptions.RequestException, + google.auth.exceptions.GoogleAuthError, ValueError) as e: + logs.error(f'V4 get_artifact_metadata failed for artifact {name}: {e}') + return None + + def download_artifact_file(self, bid: str, target: str, attempt_id: str, + name: str, output_path: str) -> bool: + """Download artifact file content using signed URL. + + Args: + bid: Android build ID. + target: Android build target name. + attempt_id: Build attempt identifier. + name: Artifact name. + output_path: Local filesystem path where the file should be saved. + + Returns: + True if download succeeded, False otherwise. + """ + resource_id = quote(name, safe='') + path = (f'/v4/builds/{bid}/{target}/attempts/{attempt_id}' + f'/artifacts/{resource_id}/url') + url = f'{self.BASE_URL}{path}' + try: + response = self._request(url) + signed_url = response.json().get('signedUrl') + except (requests.exceptions.RequestException, + google.auth.exceptions.GoogleAuthError, ValueError) as e: + logs.error(f'Error getting V4 download url for artifact {name}: {e}') + return False + + if not signed_url: + logs.error(f'V4 download url missing in response for artifact {name}') + return False + + try: + self._download_file(signed_url, output_path) + return True + except (requests.exceptions.RequestException, OSError) as e: + logs.error(f'Error downloading V4 media for artifact {name}: {e}') + return False diff --git a/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py new file mode 100644 index 00000000000..58158b0109a --- /dev/null +++ b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py @@ -0,0 +1,257 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for android_build_v4_api.py.""" + +import os +import tempfile +import unittest +from unittest import mock + +import google.auth.exceptions +import requests + +from clusterfuzz._internal.platforms.android.android_build_v4_api import \ + AndroidBuildV4Api +from clusterfuzz._internal.tests.test_libs import helpers + + +class AndroidBuildV4ApiTest(unittest.TestCase): + """Tests for AndroidBuildV4Api.""" + + def setUp(self): + helpers.patch(self, [ + 'requests.get', + ]) + + self.mock_credentials = mock.MagicMock() + self.mock_credentials.valid = True + self.mock_credentials.token = 'test_token' + + self.api = AndroidBuildV4Api.create_authenticated(self.mock_credentials) + + def test_create_authenticated_refreshes_invalid_credentials(self): + """Tests that create_authenticated refreshes invalid credentials and returns + an API client instance. + """ + mock_creds = mock.MagicMock() + mock_creds.valid = False + mock_creds.token = 'refreshed' + + client = AndroidBuildV4Api.create_authenticated(mock_creds) + mock_creds.refresh.assert_called_once() + self.assertIsInstance(client, AndroidBuildV4Api) + + def test_create_authenticated_auth_error(self): + """Tests that create_authenticated raises GoogleAuthError when credential + token refresh fails. + """ + mock_creds = mock.MagicMock() + mock_creds.valid = False + mock_creds.refresh.side_effect = ( + google.auth.exceptions.GoogleAuthError('Refresh failed')) + + with self.assertRaises(google.auth.exceptions.GoogleAuthError): + AndroidBuildV4Api.create_authenticated(mock_creds) + + def test_list_builds_success(self): + """Tests that list_builds sends correct GET request parameters and returns + parsed build data on success. + """ + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'builds': [{ + 'buildId': '123456', + 'target': { + 'name': 'target_name' + } + }] + } + self.mock.get.return_value = mock_response + + result = self.api.list_builds('git_main', 'target_name', signed=True) + self.assertEqual( + result, + {'builds': [{ + 'buildId': '123456', + 'target': { + 'name': 'target_name' + } + }]}) + self.mock.get.assert_called_once_with( + 'https://androidbuild-pa.googleapis.com/v4/builds', + headers={ + 'Authorization': 'Bearer test_token', + 'Accept': 'application/json' + }, + params={ + 'buildType': 'submitted', + 'branches': 'git_main', + 'targets': 'target_name', + 'successful': 'true', + 'pageSize': 1, + 'signed': 'true', + }, + stream=False, + timeout=60) + + def test_list_builds_error(self): + """Tests that list_builds handles HTTP errors gracefully by logging and + returning None. + """ + mock_response = mock.MagicMock() + mock_response.status_code = 500 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( + 'Server error', response=mock_response) + self.mock.get.return_value = mock_response + + result = self.api.list_builds('git_main', 'target_name') + self.assertIsNone(result) + + def test_list_builds_json_error(self): + """Tests that list_builds handles JSON decoding errors (ValueError) by + returning None. + """ + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.json.side_effect = ValueError('Invalid JSON') + self.mock.get.return_value = mock_response + + result = self.api.list_builds('git_main', 'target_name') + self.assertIsNone(result) + + def test_list_artifacts_pagination(self): + """Tests that list_artifacts follows nextPageToken pagination and + aggregates artifacts across pages. + """ + resp1 = mock.MagicMock() + resp1.status_code = 200 + resp1.json.return_value = { + 'artifacts': [{ + 'name': 'art1' + }], + 'nextPageToken': 'token123' + } + + resp2 = mock.MagicMock() + resp2.status_code = 200 + resp2.json.return_value = {'artifacts': [{'name': 'art2'}]} + + self.mock.get.side_effect = [resp1, resp2] + + artifacts = self.api.list_artifacts('123', 'target_name', regexp='.*zip') + self.assertEqual(artifacts, [{'name': 'art1'}, {'name': 'art2'}]) + + def test_list_artifacts_error(self): + """Tests that list_artifacts handles HTTP request errors gracefully by + returning an empty list. + """ + mock_response = mock.MagicMock() + mock_response.status_code = 500 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( + 'Server error', response=mock_response) + self.mock.get.return_value = mock_response + + artifacts = self.api.list_artifacts('123', 'target_name') + self.assertEqual(artifacts, []) + + def test_get_artifact_metadata(self): + """Tests that get_artifact_metadata retrieves metadata dictionary for a + specific build artifact. + """ + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + 'buildArtifactMetadata': { + 'name': 'file.zip', + 'size': '100' + } + } + self.mock.get.return_value = mock_response + + metadata = self.api.get_artifact_metadata('123', 'target', 'latest', + 'file.zip') + self.assertEqual(metadata, {'name': 'file.zip', 'size': '100'}) + + def test_get_artifact_metadata_error(self): + """Tests that get_artifact_metadata handles HTTP errors (e.g. 404) by + returning None. + """ + mock_response = mock.MagicMock() + mock_response.status_code = 404 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( + 'Not found', response=mock_response) + self.mock.get.return_value = mock_response + + metadata = self.api.get_artifact_metadata('123', 'target', 'latest', + 'file.zip') + self.assertIsNone(metadata) + + def test_download_artifact_file(self): + """Tests that download_artifact_file fetches a signed URL and streams file + chunks to local path. + """ + resp_url = mock.MagicMock() + resp_url.status_code = 200 + resp_url.json.return_value = { + 'signedUrl': 'https://storage.googleapis.com/test' + } + + resp_dl = mock.MagicMock() + resp_dl.status_code = 200 + resp_dl.iter_content.return_value = [b'chunk1', b'chunk2'] + + self.mock.get.side_effect = [resp_url, resp_dl] + + with tempfile.TemporaryDirectory() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + success = self.api.download_artifact_file('123', 'target', 'latest', + 'file.txt', output_path) + self.assertTrue(success) + with open(output_path, 'rb') as f: + self.assertEqual(f.read(), b'chunk1chunk2') + + def test_download_artifact_file_url_error(self): + """Tests that download_artifact_file handles HTTP errors during signed URL + retrieval and returns False. + """ + mock_response = mock.MagicMock() + mock_response.status_code = 500 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( + 'Server error', response=mock_response) + self.mock.get.return_value = mock_response + + success = self.api.download_artifact_file('123', 'target', 'latest', + 'file.txt', '/tmp/file.txt') + self.assertFalse(success) + + def test_download_artifact_file_os_error(self): + """Tests that download_artifact_file handles local filesystem OSError + during download and returns False. + """ + resp_url = mock.MagicMock() + resp_url.status_code = 200 + resp_url.json.return_value = { + 'signedUrl': 'https://storage.googleapis.com/test' + } + + resp_dl = mock.MagicMock() + resp_dl.status_code = 200 + resp_dl.raise_for_status.side_effect = OSError('Disk error') + + self.mock.get.side_effect = [resp_url, resp_dl] + + success = self.api.download_artifact_file('123', 'target', 'latest', + 'file.txt', '/invalid/dir/file') + self.assertFalse(success) From 21c1c34770431367c1d5aa55754be877b5117aaa Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 07:11:22 +0000 Subject: [PATCH 2/6] Fix Unit test failures --- .../tests/core/platforms/android/android_build_v4_api_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py index 58158b0109a..031e836cdf0 100644 --- a/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py +++ b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py @@ -250,7 +250,7 @@ def test_download_artifact_file_os_error(self): resp_dl.status_code = 200 resp_dl.raise_for_status.side_effect = OSError('Disk error') - self.mock.get.side_effect = [resp_url, resp_dl] + self.mock.get.side_effect = [resp_url] + [resp_dl] * 10 success = self.api.download_artifact_file('123', 'target', 'latest', 'file.txt', '/invalid/dir/file') From 83565878d9aafd550da0375c5f34bffa7ac7e856 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 18:27:57 +0000 Subject: [PATCH 3/6] adopt utils.fetch_url in AndroidBuildV4Api --- src/clusterfuzz/_internal/base/utils.py | 44 +++++- .../platforms/android/android_build_v4_api.py | 87 +++++------- .../android/android_build_v4_api_test.py | 126 ++++++++---------- 3 files changed, 123 insertions(+), 134 deletions(-) diff --git a/src/clusterfuzz/_internal/base/utils.py b/src/clusterfuzz/_internal/base/utils.py index 5cd25d55a49..4eb4b46496c 100644 --- a/src/clusterfuzz/_internal/base/utils.py +++ b/src/clusterfuzz/_internal/base/utils.py @@ -116,16 +116,46 @@ def encode_as_unicode(obj): retries=URL_REQUEST_RETRIES, delay=URL_REQUEST_FAIL_WAIT, function='base.utils.fetch_url') -def fetch_url(url): - """Fetch url content.""" - operations_timeout = environment.get_value('URL_BLOCKING_OPERATIONS_TIMEOUT') - - response = requests.get(url, timeout=operations_timeout) - if response.status_code == 404: +def fetch_url(url: str, + params: dict | None = None, + headers: dict | None = None, + request_timeout: int | float | None = None, + raise_for_not_found: bool = False, + stream: bool = False) -> str | requests.Response | None: + """Launches an HTTP GET request with retries. + + Args: + url: Target URL for the GET request. + params: Optional query parameters dictionary. + headers: Optional additional headers dictionary. + request_timeout: Optional timeout for the request in seconds. + raise_for_not_found: False by default, raise an exception for 404 + response, otherwise returns None on 404. + stream: False by default, whether to stream the response content. + + Returns: + The HTTP response text on success (or response object if stream=True), + None if 404 and raise_for_not_found is False. + + Raises: + requests.exceptions.RequestException: If the HTTP request fails or + returns an error status. + """ + if request_timeout is None: + request_timeout = environment.get_value('URL_BLOCKING_OPERATIONS_TIMEOUT') + + logs.info(f'Request for {url}', params=params, headers=headers, stream=stream) + response = requests.get( + url, + timeout=request_timeout, + params=params, + headers=headers, + stream=stream) + if not raise_for_not_found and response.status_code == 404: return None response.raise_for_status() - return response.text + return response if stream else response.text @retry.wrap( diff --git a/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py index 4b46a0ac7e7..9eb4d2bf2fa 100644 --- a/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py +++ b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py @@ -13,6 +13,7 @@ # limitations under the License. """HTTP client for Android Build API V4 REST endpoints.""" +import json import os from urllib.parse import quote @@ -21,7 +22,7 @@ from google.oauth2 import service_account import requests -from clusterfuzz._internal.base import retry +from clusterfuzz._internal.base import utils from clusterfuzz._internal.metrics import logs # HTTP request timeout in seconds. @@ -77,49 +78,8 @@ def _get_headers(self) -> dict[str, str]: 'Accept': 'application/json', } - @retry.wrap( - retries=5, delay=1, backoff=2, function='android_build_v4_api._request') - def _request(self, - url: str, - params: dict | None = None, - headers: dict | None = None, - stream: bool = False) -> requests.Response: - """Launches an HTTP GET request with retries. - - Args: - url: Target URL for the GET request. - params: Optional query parameters dictionary. - headers: Optional additional headers dictionary. - stream: Whether to stream the response content (for file downloads). - - Returns: - The HTTP response object. - - Raises: - requests.exceptions.RequestException: If the HTTP request fails or - returns an error status. - """ - request_headers = self._get_headers() - if headers: - request_headers.update(headers) - - logs.info(f'AndroidBuildV4Api request for {url}', params=params) - response = requests.get( - url, - headers=request_headers, - params=params, - stream=stream, - timeout=HTTP_TIMEOUT) - response.raise_for_status() - return response - - @retry.wrap( - retries=5, - delay=1, - backoff=2, - function='android_build_v4_api._download_file') def _download_file(self, url: str, output_path: str) -> None: - """Downloads content from a URL to output_path with retries. + """Downloads content from a URL to output_path. Args: url: Download URL. @@ -133,8 +93,11 @@ def _download_file(self, url: str, output_path: str) -> None: if dirname: os.makedirs(dirname, exist_ok=True) - response = requests.get(url, stream=True, timeout=HTTP_TIMEOUT) - response.raise_for_status() + response = utils.fetch_url( + url, + request_timeout=HTTP_TIMEOUT, + raise_for_not_found=True, + stream=True) with open(output_path, 'wb') as f: for chunk in response.iter_content(chunk_size=DEFAULT_CHUNK_SIZE): if chunk: @@ -164,8 +127,13 @@ def list_builds(self, branch: str, target: str, url = f'{self.BASE_URL}/v4/builds' try: - response = self._request(url, params=params) - return response.json() + response_text = utils.fetch_url( + url, + params=params, + headers=self._get_headers(), + request_timeout=HTTP_TIMEOUT, + raise_for_not_found=True) + return json.loads(response_text) except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: logs.error( @@ -204,8 +172,13 @@ def list_artifacts(self, params['pageToken'] = page_token try: - response = self._request(url, params=params) - result = response.json() + response_text = utils.fetch_url( + url, + params=params, + headers=self._get_headers(), + request_timeout=HTTP_TIMEOUT, + raise_for_not_found=True) + result = json.loads(response_text) except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: logs.error( @@ -239,8 +212,12 @@ def get_artifact_metadata(self, bid: str, target: str, attempt_id: str, f'/artifacts/{resource_id}') url = f'{self.BASE_URL}{path}' try: - response = self._request(url) - data = response.json() + response_text = utils.fetch_url( + url, + headers=self._get_headers(), + request_timeout=HTTP_TIMEOUT, + raise_for_not_found=True) + data = json.loads(response_text) return data.get('buildArtifactMetadata', data) except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: @@ -266,8 +243,12 @@ def download_artifact_file(self, bid: str, target: str, attempt_id: str, f'/artifacts/{resource_id}/url') url = f'{self.BASE_URL}{path}' try: - response = self._request(url) - signed_url = response.json().get('signedUrl') + response_text = utils.fetch_url( + url, + headers=self._get_headers(), + request_timeout=HTTP_TIMEOUT, + raise_for_not_found=True) + signed_url = json.loads(response_text).get('signedUrl') except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: logs.error(f'Error getting V4 download url for artifact {name}: {e}') diff --git a/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py index 031e836cdf0..f4a06c09878 100644 --- a/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py +++ b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py @@ -13,6 +13,7 @@ # limitations under the License. """Tests for android_build_v4_api.py.""" +import json import os import tempfile import unittest @@ -31,6 +32,7 @@ class AndroidBuildV4ApiTest(unittest.TestCase): def setUp(self): helpers.patch(self, [ + 'clusterfuzz._internal.base.utils.fetch_url', 'requests.get', ]) @@ -68,19 +70,17 @@ def test_list_builds_success(self): """Tests that list_builds sends correct GET request parameters and returns parsed build data on success. """ - mock_response = mock.MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { + self.mock.fetch_url.return_value = json.dumps({ 'builds': [{ 'buildId': '123456', 'target': { 'name': 'target_name' } }] - } - self.mock.get.return_value = mock_response + }) - result = self.api.list_builds('git_main', 'target_name', signed=True) + result = self.api.list_builds( + branch='git_main', target='target_name', signed=True) self.assertEqual( result, {'builds': [{ @@ -89,7 +89,7 @@ def test_list_builds_success(self): 'name': 'target_name' } }]}) - self.mock.get.assert_called_once_with( + self.mock.fetch_url.assert_called_once_with( 'https://androidbuild-pa.googleapis.com/v4/builds', headers={ 'Authorization': 'Bearer test_token', @@ -103,18 +103,15 @@ def test_list_builds_success(self): 'pageSize': 1, 'signed': 'true', }, - stream=False, - timeout=60) + request_timeout=60, + raise_for_not_found=True) def test_list_builds_error(self): """Tests that list_builds handles HTTP errors gracefully by logging and returning None. """ - mock_response = mock.MagicMock() - mock_response.status_code = 500 - mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( - 'Server error', response=mock_response) - self.mock.get.return_value = mock_response + self.mock.fetch_url.side_effect = requests.exceptions.HTTPError( + 'Server error') result = self.api.list_builds('git_main', 'target_name') self.assertIsNone(result) @@ -123,62 +120,49 @@ def test_list_builds_json_error(self): """Tests that list_builds handles JSON decoding errors (ValueError) by returning None. """ - mock_response = mock.MagicMock() - mock_response.status_code = 200 - mock_response.json.side_effect = ValueError('Invalid JSON') - self.mock.get.return_value = mock_response + self.mock.fetch_url.return_value = 'invalid json' - result = self.api.list_builds('git_main', 'target_name') + result = self.api.list_builds(branch='git_main', target='target_name') self.assertIsNone(result) def test_list_artifacts_pagination(self): """Tests that list_artifacts follows nextPageToken pagination and aggregates artifacts across pages. """ - resp1 = mock.MagicMock() - resp1.status_code = 200 - resp1.json.return_value = { + resp1 = json.dumps({ 'artifacts': [{ 'name': 'art1' }], 'nextPageToken': 'token123' - } - - resp2 = mock.MagicMock() - resp2.status_code = 200 - resp2.json.return_value = {'artifacts': [{'name': 'art2'}]} + }) + resp2 = json.dumps({'artifacts': [{'name': 'art2'}]}) - self.mock.get.side_effect = [resp1, resp2] + self.mock.fetch_url.side_effect = [resp1, resp2] - artifacts = self.api.list_artifacts('123', 'target_name', regexp='.*zip') + artifacts = self.api.list_artifacts( + bid='123', target='target_name', regexp='.*zip') self.assertEqual(artifacts, [{'name': 'art1'}, {'name': 'art2'}]) def test_list_artifacts_error(self): """Tests that list_artifacts handles HTTP request errors gracefully by returning an empty list. """ - mock_response = mock.MagicMock() - mock_response.status_code = 500 - mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( - 'Server error', response=mock_response) - self.mock.get.return_value = mock_response + self.mock.fetch_url.side_effect = requests.exceptions.HTTPError( + 'Server error') - artifacts = self.api.list_artifacts('123', 'target_name') + artifacts = self.api.list_artifacts(bid='123', target='target_name') self.assertEqual(artifacts, []) def test_get_artifact_metadata(self): """Tests that get_artifact_metadata retrieves metadata dictionary for a specific build artifact. """ - mock_response = mock.MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { + self.mock.fetch_url.return_value = json.dumps({ 'buildArtifactMetadata': { 'name': 'file.zip', 'size': '100' } - } - self.mock.get.return_value = mock_response + }) metadata = self.api.get_artifact_metadata('123', 'target', 'latest', 'file.zip') @@ -188,36 +172,31 @@ def test_get_artifact_metadata_error(self): """Tests that get_artifact_metadata handles HTTP errors (e.g. 404) by returning None. """ - mock_response = mock.MagicMock() - mock_response.status_code = 404 - mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( - 'Not found', response=mock_response) - self.mock.get.return_value = mock_response + self.mock.fetch_url.side_effect = requests.exceptions.HTTPError('Not found') - metadata = self.api.get_artifact_metadata('123', 'target', 'latest', - 'file.zip') + metadata = self.api.get_artifact_metadata( + bid='123', target='target', attempt_id='latest', name='file.zip') self.assertIsNone(metadata) def test_download_artifact_file(self): """Tests that download_artifact_file fetches a signed URL and streams file chunks to local path. """ - resp_url = mock.MagicMock() - resp_url.status_code = 200 - resp_url.json.return_value = { - 'signedUrl': 'https://storage.googleapis.com/test' - } - + resp_url = json.dumps({'signedUrl': 'https://storage.googleapis.com/test'}) resp_dl = mock.MagicMock() resp_dl.status_code = 200 resp_dl.iter_content.return_value = [b'chunk1', b'chunk2'] - self.mock.get.side_effect = [resp_url, resp_dl] + self.mock.fetch_url.side_effect = [resp_url, resp_dl] with tempfile.TemporaryDirectory() as temp_dir: output_path = os.path.join(temp_dir, 'output.txt') - success = self.api.download_artifact_file('123', 'target', 'latest', - 'file.txt', output_path) + success = self.api.download_artifact_file( + bid='123', + target='target', + attempt_id='latest', + name='file.txt', + output_path=output_path) self.assertTrue(success) with open(output_path, 'rb') as f: self.assertEqual(f.read(), b'chunk1chunk2') @@ -226,32 +205,31 @@ def test_download_artifact_file_url_error(self): """Tests that download_artifact_file handles HTTP errors during signed URL retrieval and returns False. """ - mock_response = mock.MagicMock() - mock_response.status_code = 500 - mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( - 'Server error', response=mock_response) - self.mock.get.return_value = mock_response - - success = self.api.download_artifact_file('123', 'target', 'latest', - 'file.txt', '/tmp/file.txt') + self.mock.fetch_url.side_effect = requests.exceptions.HTTPError( + 'Server error') + + success = self.api.download_artifact_file( + bid='123', + target='target', + attempt_id='latest', + name='file.txt', + output_path='/tmp/file.txt') self.assertFalse(success) def test_download_artifact_file_os_error(self): """Tests that download_artifact_file handles local filesystem OSError during download and returns False. """ - resp_url = mock.MagicMock() - resp_url.status_code = 200 - resp_url.json.return_value = { - 'signedUrl': 'https://storage.googleapis.com/test' - } - + resp_url = json.dumps({'signedUrl': 'https://storage.googleapis.com/test'}) resp_dl = mock.MagicMock() resp_dl.status_code = 200 - resp_dl.raise_for_status.side_effect = OSError('Disk error') - self.mock.get.side_effect = [resp_url] + [resp_dl] * 10 + self.mock.fetch_url.side_effect = [resp_url, resp_dl] - success = self.api.download_artifact_file('123', 'target', 'latest', - 'file.txt', '/invalid/dir/file') + success = self.api.download_artifact_file( + bid='123', + target='target', + attempt_id='latest', + name='file.txt', + output_path='/invalid/dir/file') self.assertFalse(success) From a5950084c81087d8ef35eff079fff523cd83a667 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Tue, 21 Jul 2026 19:06:36 +0000 Subject: [PATCH 4/6] Fix UT failure --- .../android/android_build_v4_api_test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py index f4a06c09878..3798c12ff94 100644 --- a/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py +++ b/src/clusterfuzz/_internal/tests/core/platforms/android/android_build_v4_api_test.py @@ -223,13 +223,16 @@ def test_download_artifact_file_os_error(self): resp_url = json.dumps({'signedUrl': 'https://storage.googleapis.com/test'}) resp_dl = mock.MagicMock() resp_dl.status_code = 200 + resp_dl.iter_content.side_effect = OSError('Disk error') self.mock.fetch_url.side_effect = [resp_url, resp_dl] - success = self.api.download_artifact_file( - bid='123', - target='target', - attempt_id='latest', - name='file.txt', - output_path='/invalid/dir/file') - self.assertFalse(success) + with tempfile.TemporaryDirectory() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + success = self.api.download_artifact_file( + bid='123', + target='target', + attempt_id='latest', + name='file.txt', + output_path=output_path) + self.assertFalse(success) From 1e3a245fb01d5860ebbd18eba6cf81dd22330e06 Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Thu, 23 Jul 2026 10:10:50 -0600 Subject: [PATCH 5/6] [Android v4 API] Migrate fetch_artifact to use new V4 API (#5371) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Overview Bug: b/422775458 Our previous implementation of the android builv v4 api was wrong, it failed always because the api no longer worked on discovery service url. In other [PR](https://github.com/google/clusterfuzz/pull/5370) we focused on creating the new api to interact with the v4 requirements. In this PR we focus on using this api class in the `fetch_artifact.py` module, ## Changes - Made private all the methods that were only used internally in `fetch_artifact.py` - Refactors previous usage of the v4 api to now use the current version with minimal changes in the `fetch_artifact.py` script to avoid breaking anything ## Tests performed - Created unit tests for `feth_artifact.py` - Tested this script using [a custom script](https://github.com/google/clusterfuzz/compare/android-build-v4-testing) (test details in [this PR description](https://github.com/google/clusterfuzz/pull/5370) ) #### PR review 1. [**#5370** Add Android Build API V4 REST client and tests ](https://github.com/google/clusterfuzz/pull/5370) 2. 👉 #5371 Migrate fetch_artifact to use new V4 API *(This PR)* 3. [#5373 Creates Feature Flag for Android V4 API](https://github.com/google/clusterfuzz/pull/5373) Please only approve, I'll merge this PR's to avoid any possible merge conflict that can appear in the process :D --- .../_internal/base/feature_flags.py | 1 + .../platforms/android/android_build_v4_api.py | 5 +- .../platforms/android/fetch_artifact.py | 258 ++++++++++-------- .../_internal/platforms/android/flash.py | 7 +- .../platforms/android/symbols_downloader.py | 11 +- .../platforms/android/fetch_artifact_test.py | 174 ++++++++++++ 6 files changed, 336 insertions(+), 120 deletions(-) create mode 100644 src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py diff --git a/src/clusterfuzz/_internal/base/feature_flags.py b/src/clusterfuzz/_internal/base/feature_flags.py index 5dfeccae1c1..9660745990c 100644 --- a/src/clusterfuzz/_internal/base/feature_flags.py +++ b/src/clusterfuzz/_internal/base/feature_flags.py @@ -44,6 +44,7 @@ class FeatureFlags(Enum): ENABLE_FUZZ_FOR_BOTS = 'enable_fuzz_for_bots' STORAGE_THREADED_OPS_FUZZ_TARGETS = 'storage_threaded_ops_fuzz_targets' + CALL_ANDROID_API = 'call_android_api' @property def flag(self): diff --git a/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py index 9eb4d2bf2fa..cd605efe6d6 100644 --- a/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py +++ b/src/clusterfuzz/_internal/platforms/android/android_build_v4_api.py @@ -133,7 +133,10 @@ def list_builds(self, branch: str, target: str, headers=self._get_headers(), request_timeout=HTTP_TIMEOUT, raise_for_not_found=True) - return json.loads(response_text) + data = json.loads(response_text) + if data.get('builds') and len(data['builds']) > 0: + return data + return None except (requests.exceptions.RequestException, google.auth.exceptions.GoogleAuthError, ValueError) as e: logs.error( diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index e86cced06c6..961e012bc50 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -23,21 +23,28 @@ from typing import Optional import apiclient +from google.oauth2 import service_account from oauth2client.service_account import ServiceAccountCredentials +from clusterfuzz._internal.base import feature_flags from clusterfuzz._internal.config import db_config from clusterfuzz._internal.google_cloud_utils import storage from clusterfuzz._internal.metrics import logs +from clusterfuzz._internal.platforms.android.android_build_v4_api import \ + AndroidBuildV4Api from clusterfuzz._internal.system import environment -from . import adb - # 20 MB default chunk size. DEFAULT_CHUNK_SIZE = 20 * 1024 * 1024 # Maximum number of retries for artifact access. MAX_RETRIES = 5 +ANDROID_BUILD_API_SCOPES = [ + 'https://www.googleapis.com/auth/androidbuild.internal', + 'https://www.googleapis.com/auth/cloud-platform' +] + STABLE_CUTTLEFISH_BUILD = { 'bid': '11655237', 'branch': 'git_main', @@ -64,7 +71,21 @@ def _use_v4(): return False -def execute_request_with_retries(request): +def _call_android_api_enabled(): + """Return True if we should call the Android Build API, enabled by default, + Disabled always if invoked in a uworker + """ + if environment.is_uworker(): + logs.info('AndroidBuildAPI access disabled for uworker.') + return False + + flag = feature_flags.FeatureFlags.CALL_ANDROID_API.flag + if flag is None: + return True + return flag.enabled + + +def _execute_request_with_retries(request): """Executes request and retries on failure.""" result = None for _ in range(MAX_RETRIES): @@ -77,8 +98,8 @@ def execute_request_with_retries(request): return result -def download_artifact(client, bid, target, attempt_id, name, output_directory, - output_filename): +def _download_artifact(client, bid, target, attempt_id, name, output_directory, + output_filename): """Download one artifact.""" logs.info('reached download_artifact') logs.info('artifact to download: %s' % name) @@ -96,12 +117,12 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, artifact_name=name) if _use_v4(): - artifact_query = client.buildartifacts().get( - buildId=bid, target=target, attemptId=attempt_id, resourceId=name) + artifact = client.get_artifact_metadata(bid, target, attempt_id, name) else: artifact_query = client.buildartifact().get( buildId=bid, target=target, attemptId=attempt_id, resourceId=name) - artifact = execute_request_with_retries(artifact_query) + artifact = _execute_request_with_retries(artifact_query) + if artifact is None: logs.error( 'AndroidBuildAPI download_artifact failed: artifact metadata ' @@ -131,22 +152,6 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, if size >= DEFAULT_CHUNK_SIZE: chunksize = DEFAULT_CHUNK_SIZE - # Just like get, except get_media. - logs.info( - 'AndroidBuildAPI download_artifact media download started.', - api_version=version_tag, - operation='download_artifact_media', - build_id=bid, - target=target, - attempt_id=attempt_id, - artifact_name=name) - if _use_v4(): - dl_request = client.buildartifacts().get_media( - buildId=bid, target=target, attemptId=attempt_id, resourceId=name) - else: - dl_request = client.buildartifact().get_media( - buildId=bid, target=target, attemptId=attempt_id, resourceId=name) - if output_filename: file_name = output_filename else: @@ -172,20 +177,50 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, logs.info('Output dir: %s' % output_dir) if not os.path.exists(output_dir): logs.info(f'Creating directory {output_dir}') - os.mkdir(output_dir) + os.makedirs(output_dir, exist_ok=True) - with io.FileIO(output_path, mode='wb') as file_handle: - downloader = apiclient.http.MediaIoBaseDownload( - file_handle, dl_request, chunksize=chunksize) - done = False + # TODO(b/537368595) Remove unnecesary logging. + # Just like get, except get_media. + logs.info( + 'AndroidBuildAPI download_artifact media download started.', + api_version=version_tag, + operation='download_artifact_media', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name) + + if _use_v4(): + success = client.download_artifact_file(bid, target, attempt_id, name, + output_path) + if not success: + logs.error( + 'AndroidBuildAPI download_artifact failed for V4.', + api_version=version_tag, + operation='download_artifact', + build_id=bid, + target=target, + attempt_id=attempt_id, + artifact_name=name, + output_path=output_path, + status='failed') + return None + else: + dl_request = client.buildartifact().get_media( + buildId=bid, target=target, attemptId=attempt_id, resourceId=name) - while not done: - status, done = downloader.next_chunk() - if status: - size_completed = int(status.resumable_progress) - if size != 0: - percent_completed = (size_completed * 100.0) / size - logs.info('%.1f%% complete.' % percent_completed) + with io.FileIO(output_path, mode='wb') as file_handle: + downloader = apiclient.http.MediaIoBaseDownload( + file_handle, dl_request, chunksize=chunksize) + done = False + + while not done: + status, done = downloader.next_chunk() + if status: + size_completed = int(status.resumable_progress) + if size != 0: + percent_completed = (size_completed * 100.0) / size + logs.info('%.1f%% complete.' % percent_completed) logs.info( 'AndroidBuildAPI download_artifact completed successfully.', @@ -200,12 +235,19 @@ def download_artifact(client, bid, target, attempt_id, name, output_directory, return output_path -def get_artifacts_for_build(client, - bid: str, - target: str, - attempt_id: str = 'latest', - regexp: Optional[str] = None) -> List[str]: +def _get_artifacts_for_build(client, + bid: str, + target: str, + attempt_id: str = 'latest', + regexp: Optional[str] = None) -> List[str]: """Return list of artifacts for a given build.""" + if not regexp: + logs.warning( + 'Regexp is empty, returning early to avoid querying all artifacts.', + bid=bid, + target=target) + return [] + version_tag = 'V4' if _use_v4() else 'V3' logs.info( 'AndroidBuildAPI get_artifacts_for_build started.', @@ -216,46 +258,27 @@ def get_artifacts_for_build(client, attempt_id=attempt_id, regexp=regexp) - if _use_v4(): - if not regexp: - request = client.buildartifacts().list( - buildId=bid, target=target, attemptId=attempt_id) - else: - request = client.buildartifacts().list( - buildId=bid, - target=target, - attemptId=attempt_id, - nameRegexp=regexp, - maxResults=100) - else: - if not regexp: - request = client.buildartifact().list( - buildId=bid, target=target, attemptId=attempt_id) - else: - request = client.buildartifact().list( - buildId=bid, - target=target, - attemptId=attempt_id, - nameRegexp=regexp, - maxResults=100) - - request_str = (f'{request.uri}, {request.method}, ' - f'{request.body}, {request.methodId}') - artifacts = [] - results = [] - while request: - result = execute_request_with_retries(request) - if not result: - break - results.append(result) - if result and 'artifacts' in result: - for artifact in result['artifacts']: - artifacts.append(artifact) - if _use_v4(): - request = client.buildartifacts().list_next(request, result) - else: + + if _use_v4(): + artifacts = client.list_artifacts(bid, target, attempt_id, regexp=regexp) + else: + request = client.buildartifact().list( + buildId=bid, + target=target, + attemptId=attempt_id, + nameRegexp=regexp, + maxResults=100) + + while request: + result = _execute_request_with_retries(request) + if not result: + break + results.append(result) + if result and 'artifacts' in result: + for artifact in result['artifacts']: + artifacts.append(artifact) request = client.buildartifact().list_next(request, result) logs.info( @@ -271,13 +294,12 @@ def get_artifacts_for_build(client, if not artifacts: logs.error(f'No artifact found for target {target}, build id {bid}.\n' - f'request {request_str}, results {results}') - adb.bad_state_reached() + f'results {results}') return artifacts -def get_client(): +def _get_client(): """Return client with connection to build apiary.""" # Connect using build apiary service account credentials. build_apiary_service_account_private_key = db_config.get_value( @@ -287,22 +309,23 @@ def get_client(): 'Android build apiary credentials are not set, skip artifact fetch.') return None - credentials = ServiceAccountCredentials.from_json_keyfile_dict( - json.loads(build_apiary_service_account_private_key), - scopes='https://www.googleapis.com/auth/androidbuild.internal') + key_dict = json.loads(build_apiary_service_account_private_key) + + logs.info( + 'AndroidBuildAPI client initialization started.', + api_version='V4' if _use_v4() else 'V3') + if _use_v4(): - logs.info( - 'AndroidBuildAPI client initialization started.', api_version='V4') - client = apiclient.discovery.build( - 'androidbuildinternal', - 'v4', - discoveryServiceUrl= - 'https://androidbuild-pa.googleapis.com/$discovery/rest?version=v4', - credentials=credentials, - static_discovery=False) + try: + credentials = service_account.Credentials.from_service_account_info( + key_dict, scopes=ANDROID_BUILD_API_SCOPES) + client = AndroidBuildV4Api.create_authenticated(credentials) + except Exception as e: + logs.error(f'Failed to initialize AndroidBuildV4Api: {e}') + return None else: - logs.info( - 'AndroidBuildAPI client initialization started.', api_version='V3') + credentials = ServiceAccountCredentials.from_json_keyfile_dict( + key_dict, scopes=ANDROID_BUILD_API_SCOPES) client = apiclient.discovery.build( 'androidbuildinternal', 'v3', @@ -312,7 +335,7 @@ def get_client(): return client -def get_stable_build_info(): +def _get_stable_build_info(): """Return stable artifact for cuttlefish branch and target.""" logs.info('Reached get_stable_build_info') stable_build_info = STABLE_CUTTLEFISH_BUILD @@ -332,14 +355,19 @@ def get_stable_build_info(): def get_latest_artifact_info(branch, target, signed=False, stable_build=False): """Return latest artifact for a branch and target.""" - client = get_client() + if not _call_android_api_enabled(): + logs.warning( + 'Android build API is disabled by feature flag call_android_api.') + return None + + client = _get_client() if not client: return None # TODO(https://github.com/google/clusterfuzz/issues/3950) # After stabilizing the Cuttlefish image, revert this if environment.is_android_cuttlefish() and stable_build: - build_info = get_stable_build_info() + build_info = _get_stable_build_info() # Use tip-of-tree build if 'bid' is missing or 0. # Setting 'bid' to 0 in stable_build_info.json # allows for easy switching between stable build @@ -355,14 +383,9 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): branch=branch, target=target, signed=signed) + if _use_v4(): - request = client.builds().list( # pylint: disable=no-member - buildType='submitted', - branch=branch, - target=target, - successful=True, - maxResults=1, - signed=signed) + builds = client.list_builds(branch, target, signed) else: request = client.build().list( # pylint: disable=no-member buildType='submitted', @@ -371,10 +394,9 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): successful=True, maxResults=1, signed=signed) - request_str = (f'{request.uri}, {request.method}, ' - f'{request.body}, {request.methodId}') + res = _execute_request_with_retries(request) + builds = res if (res and res.get('builds')) else None - builds = execute_request_with_retries(request) if not builds: logs.error( 'AndroidBuildAPI get_latest_artifact_info failed: no builds found.', @@ -383,8 +405,7 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): branch=branch, target=target, signed=signed, - status='failed', - request_str=request_str) + status='failed') return None build = builds['builds'][0] @@ -405,12 +426,17 @@ def get_latest_artifact_info(branch, target, signed=False, stable_build=False): def get(bid, target, regex, output_directory, output_filename=None): """Return artifact for a given build id, target and file regex.""" - client = get_client() + if not _call_android_api_enabled(): + logs.warning( + 'Android build API is disabled by feature flag call_android_api.') + return None + + client = _get_client() if not client: return None # Run the script to fetch the artifact. - return run_script( + return _run_script( client=client, bid=bid, target=target, @@ -419,9 +445,9 @@ def get(bid, target, regex, output_directory, output_filename=None): output_filename=output_filename) -def run_script(client, bid, target, regex, output_directory, output_filename): +def _run_script(client, bid, target, regex, output_directory, output_filename): """Download artifacts as specified.""" - artifacts = get_artifacts_for_build( + artifacts = _get_artifacts_for_build( client=client, bid=bid, target=target, attempt_id='latest', regexp=regex) if not artifacts: logs.error(f'Artifact could not be fetched for target {target}, ' @@ -439,7 +465,7 @@ def run_script(client, bid, target, regex, output_directory, output_filename): continue if regex.match(artifact_name): - loop_result = download_artifact( + loop_result = _download_artifact( client=client, bid=bid, target=target, diff --git a/src/clusterfuzz/_internal/platforms/android/flash.py b/src/clusterfuzz/_internal/platforms/android/flash.py index b292b1491e4..283a80120d3 100644 --- a/src/clusterfuzz/_internal/platforms/android/flash.py +++ b/src/clusterfuzz/_internal/platforms/android/flash.py @@ -85,7 +85,7 @@ def download_latest_build(build_info, image_regexes, image_directory): logs.error('Failed to download artifact %s for ' 'branch %s and target %s.' % (image_file_paths, build_info['branch'], target)) - return + adb.bad_state_reached() for file_path in image_file_paths: if file_path.endswith('.zip') or file_path.endswith('.tar.gz'): @@ -97,7 +97,10 @@ def boot_stable_build_cuttlefish(branch, target, image_directory): """Boot cuttlefish instance using stable build id fetched from gcs.""" build_info = fetch_artifact.get_latest_artifact_info( branch, target, stable_build=True) - download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) + if not build_info: + logs.error('Unable to fetch stable build info for cuttlefish.') + else: + download_latest_build(build_info, FLASH_CUTTLEFISH_REGEXES, image_directory) adb.recreate_cuttlefish_device() adb.connect_to_cuttlefish_device() diff --git a/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py b/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py index 112d86fc215..55a906b5498 100644 --- a/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py +++ b/src/clusterfuzz/_internal/platforms/android/symbols_downloader.py @@ -196,7 +196,12 @@ def download_trusty_symbols_if_needed(symbols_directory, app_name, bid): branch = 'polygon-trusty-whitechapel-master' if not bid: - bid = fetch_artifact.get_latest_artifact_info(branch, ab_target)['bid'] + build_info = fetch_artifact.get_latest_artifact_info(branch, ab_target) + if not build_info: + logs.error(f'Unable to fetch build info for branch {branch} ' + f'and target {ab_target}.') + return + bid = build_info['bid'] artifact_filename = f'{ab_target}-{bid}.syms.zip' symbols_archive_path = os.path.join(symbols_directory, artifact_filename) @@ -204,6 +209,10 @@ def download_trusty_symbols_if_needed(symbols_directory, app_name, bid): download_artifact_if_needed(bid, symbols_directory, symbols_archive_path, [ab_target], artifact_filename, None) + if not os.path.exists(symbols_archive_path): + logs.error(f'Unable to locate symbols archive {symbols_archive_path}.') + return + with zipfile.ZipFile(symbols_archive_path, 'r') as symbols_zipfile: for filepath in symbols_zipfile.namelist(): if f'{app_name}.syms.elf' in filepath: diff --git a/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py b/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py new file mode 100644 index 00000000000..5417a64e229 --- /dev/null +++ b/src/clusterfuzz/_internal/tests/core/platforms/android/fetch_artifact_test.py @@ -0,0 +1,174 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for fetch_artifact.py.""" + +# pylint: disable=protected-access + +import unittest +from unittest import mock + +from clusterfuzz._internal.base import feature_flags +from clusterfuzz._internal.platforms.android import fetch_artifact +from clusterfuzz._internal.tests.test_libs import helpers + + +class FetchArtifactTest(unittest.TestCase): + """Tests for fetch_artifact.""" + + def setUp(self): + helpers.patch(self, [ + 'clusterfuzz._internal.platforms.android.fetch_artifact._get_client', + 'clusterfuzz._internal.platforms.android.fetch_artifact._use_v4', + 'clusterfuzz._internal.platforms.android.fetch_artifact._call_android_api_enabled', + 'clusterfuzz._internal.platforms.android.fetch_artifact._execute_request_with_retries', + ]) + self.mock_client = mock.MagicMock() + self.mock._get_client.return_value = self.mock_client + self.mock._call_android_api_enabled.return_value = True + + def test_get_latest_artifact_v4_success(self): + """Tests get_latest_artifact_info (V4). Expects extraction of {bid, branch, target} when list_builds returns data.""" + self.mock._use_v4.return_value = True + self.mock_client.list_builds.return_value = { + 'builds': [{ + 'buildId': '123', + 'target': { + 'name': 'test_target' + } + }] + } + + result = fetch_artifact.get_latest_artifact_info('branch1', 'target1') + self.assertEqual(result, { + 'bid': '123', + 'branch': 'branch1', + 'target': 'test_target' + }) + self.mock_client.list_builds.assert_called_once_with( + 'branch1', 'target1', False) + + def test_get_latest_artifact_v4_failure_no_builds(self): + """Tests get_latest_artifact_info (V4) returns None gracefully when no builds are found.""" + self.mock._use_v4.return_value = True + self.mock_client.list_builds.return_value = {} + + result = fetch_artifact.get_latest_artifact_info('branch1', 'target1') + self.assertIsNone(result) + + def test_get_latest_artifact_v3_success(self): + """Tests get_latest_artifact_info (V3). Expects extraction of {bid, branch, target} via legacy HTTP execution.""" + self.mock._use_v4.return_value = False + + mock_request = mock.MagicMock() + self.mock_client.build().list.return_value = mock_request + + self.mock._execute_request_with_retries.return_value = { + 'builds': [{ + 'buildId': '456', + 'target': { + 'name': 'test_target2' + } + }] + } + + result = fetch_artifact.get_latest_artifact_info( + 'branch2', 'target2', signed=True) + self.assertEqual(result, { + 'bid': '456', + 'branch': 'branch2', + 'target': 'test_target2' + }) + + self.mock_client.build().list.assert_called_once_with( + buildType='submitted', + branch='branch2', + target='target2', + successful=True, + maxResults=1, + signed=True) + self.mock._execute_request_with_retries.assert_called_once_with( + mock_request) + + def test_get_latest_artifact_v3_failure_no_builds(self): + """Tests get_latest_artifact_info (V3) returns None gracefully when the payload is empty.""" + self.mock._use_v4.return_value = False + + mock_request = mock.MagicMock() + self.mock_client.build().list.return_value = mock_request + self.mock._execute_request_with_retries.return_value = {'builds': []} + + result = fetch_artifact.get_latest_artifact_info( + 'branch2', 'target2', signed=True) + self.assertIsNone(result) + + def test_get_latest_artifact_client_auth_failure(self): + """Tests get_latest_artifact_info exits early and returns None when client auth fails.""" + self.mock._get_client.return_value = None + + result = fetch_artifact.get_latest_artifact_info('branch1', 'target1') + self.assertIsNone(result) + + def test_get_latest_artifact_disabled_by_feature_flag(self): + """Tests get_latest_artifact_info exits early and returns None when disabled by feature flag.""" + self.mock._call_android_api_enabled.return_value = False + + result = fetch_artifact.get_latest_artifact_info('branch1', 'target1') + self.assertIsNone(result) + + def test_get_artifacts_for_build_empty_regexp(self): + """Tests _get_artifacts_for_build returns [] returning early when regexp is empty, bypassing API calls.""" + result = fetch_artifact._get_artifacts_for_build( + self.mock_client, 'bid', 'target', regexp='') + self.assertEqual(result, []) + self.mock_client.list_artifacts.assert_not_called() + self.mock_client.buildartifact().list.assert_not_called() + + +class CallAndroidApiEnabledTest(unittest.TestCase): + """Tests for _call_android_api_enabled.""" + + def setUp(self): + helpers.patch(self, [ + 'clusterfuzz._internal.system.environment.is_uworker', + ]) + self.mock.is_uworker.return_value = False + + def test_is_uworker_returns_false(self): + self.mock.is_uworker.return_value = True + self.assertFalse(fetch_artifact._call_android_api_enabled()) + + def test_flag_none_returns_true(self): + with mock.patch.object( + feature_flags.FeatureFlags, 'flag', + new_callable=mock.PropertyMock) as mock_flag: + mock_flag.return_value = None + self.assertTrue(fetch_artifact._call_android_api_enabled()) + + def test_flag_enabled_returns_true(self): + mock_flag_obj = mock.MagicMock() + mock_flag_obj.enabled = True + with mock.patch.object( + feature_flags.FeatureFlags, 'flag', + new_callable=mock.PropertyMock) as mock_flag: + mock_flag.return_value = mock_flag_obj + self.assertTrue(fetch_artifact._call_android_api_enabled()) + + def test_flag_disabled_returns_false(self): + mock_flag_obj = mock.MagicMock() + mock_flag_obj.enabled = False + with mock.patch.object( + feature_flags.FeatureFlags, 'flag', + new_callable=mock.PropertyMock) as mock_flag: + mock_flag.return_value = mock_flag_obj + self.assertFalse(fetch_artifact._call_android_api_enabled()) From 89d6ea2909378c7d15b57ba07481c326e539f32e Mon Sep 17 00:00:00 2001 From: Ivan Barba Date: Thu, 23 Jul 2026 16:29:59 +0000 Subject: [PATCH 6/6] Nit: simplify return in _call_android_api_enabled --- src/clusterfuzz/_internal/platforms/android/fetch_artifact.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py index 961e012bc50..362ccd6b7e3 100644 --- a/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py +++ b/src/clusterfuzz/_internal/platforms/android/fetch_artifact.py @@ -80,9 +80,7 @@ def _call_android_api_enabled(): return False flag = feature_flags.FeatureFlags.CALL_ANDROID_API.flag - if flag is None: - return True - return flag.enabled + return flag.enabled if flag else True def _execute_request_with_retries(request):