Skip to content

test: add comprehensive tests for gemini-cli skills setup #401

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
IsmailMehdi merged 7 commits into from
May 26, 2026
Merged

test: add comprehensive tests for gemini-cli skills setup #401

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a7cc976
test: add gemini cli skill content preservation test
May 21, 2026
6c85e12
Merge branch 'GoogleCloudPlatform:main' into unit_test_gemini_cli
minzznguyen May 26, 2026
35d1d73
test: add more integration tests for gemini cli skills
May 26, 2026
a082b8c
refactor: simplify skill content preservation test
May 26, 2026
a8ffc38
style: remove docstrings and comments from tests
May 26, 2026
f75aa08
Merge branch 'main' into unit_test_gemini_cli
IsmailMehdi May 26, 2026
81d7872
style: fix trailing whitespaces on blank lines
May 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
339 changes: 339 additions & 0 deletions evalbench/test/gemini_cli_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,339 @@
import os
import sys
from unittest.mock import MagicMock, patch

import pytest

sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from generators.models.gemini_cli import GeminiCliGenerator
Comment thread
IsmailMehdi marked this conversation as resolved.


@patch('generators.models.gemini_cli.subprocess.run')
@patch('generators.models.gemini_cli.os.path.exists')
@patch('generators.models.gemini_cli.shutil.copytree')
@patch('generators.models.gemini_cli.shutil.rmtree')
def test_setup_single_skill_string(
mock_rmtree,
mock_copytree,
mock_exists,
mock_run,
monkeypatch,
):
real_home = "/fake/real_home"
real_skill_path = os.path.join(real_home, ".gemini", "skills", "my-single-skill")

mock_exists.side_effect = lambda path: path == real_skill_path
mock_run.return_value = MagicMock(returncode=0, stdout="fake-token")

config = {"setup": {"skills": ["my-single-skill"]}}

monkeypatch.setenv("HOME", real_home)
with (
patch('generators.models.gemini_cli.os.makedirs'),
patch('generators.models.gemini_cli.open', create=True),
):
GeminiCliGenerator(config)

expected_fake_home = os.path.abspath(os.path.join(".venv", "fake_home"))
expected_fake_skill_path = os.path.join(
expected_fake_home, ".gemini", "skills", "my-single-skill"
)
mock_copytree.assert_called_once_with(real_skill_path, expected_fake_skill_path)


def test_skill_content_preserved(tmp_path, monkeypatch):
real_home = tmp_path / "real_home"
real_home.mkdir()

skill_name = "my-content-skill"
real_skill_dir = real_home / ".gemini" / "skills" / skill_name
real_skill_dir.mkdir(parents=True)

skill_file = real_skill_dir / "secret.txt"
expected_content = "hello world"
skill_file.write_text(expected_content)

monkeypatch.chdir(tmp_path)
monkeypatch.setenv("HOME", str(real_home))

generator = GeminiCliGenerator({})

generator._setup_skills([skill_name])

expected_fake_skill_file = (
tmp_path / ".venv" / "fake_home" / ".gemini" / "skills" / skill_name / "secret.txt"
)
assert expected_fake_skill_file.exists(), (
f"Copied skill file not found at {expected_fake_skill_file}"
)
assert expected_fake_skill_file.read_text() == expected_content


@patch('generators.models.gemini_cli.subprocess.run')
@patch('generators.models.gemini_cli.os.path.exists')
@patch('generators.models.gemini_cli.shutil.copytree')
@patch('generators.models.gemini_cli.shutil.rmtree')
def test_setup_multiple_skills_string(
mock_rmtree,
mock_copytree,
mock_exists,
mock_run,
monkeypatch,
):
real_home = "/fake/real_home"
skill_a_path = os.path.join(real_home, ".gemini", "skills", "skill-A")
skill_b_path = os.path.join(real_home, ".gemini", "skills", "skill-B")

mock_exists.side_effect = lambda path: path in (skill_a_path, skill_b_path)
mock_run.return_value = MagicMock(returncode=0, stdout="fake-token")

config = {"setup": {"skills": ["skill-A", "skill-B"]}}

monkeypatch.setenv("HOME", real_home)
with (
patch('generators.models.gemini_cli.os.makedirs'),
patch('generators.models.gemini_cli.open', create=True),
):
GeminiCliGenerator(config)

expected_fake_home = os.path.abspath(os.path.join(".venv", "fake_home"))
expected_fake_a = os.path.join(expected_fake_home, ".gemini", "skills", "skill-A")
expected_fake_b = os.path.join(expected_fake_home, ".gemini", "skills", "skill-B")

assert mock_copytree.call_count == 2
mock_copytree.assert_any_call(skill_a_path, expected_fake_a)
mock_copytree.assert_any_call(skill_b_path, expected_fake_b)


@patch('generators.models.gemini_cli.subprocess.run')
@patch('generators.models.gemini_cli.os.path.exists')
def test_setup_skill_dict_link(
mock_exists,
mock_run,
monkeypatch,
):
real_home = "/fake/real_home"
mock_exists.return_value = False
mock_run.return_value = MagicMock(returncode=0, stdout="success")

config = {
"gemini_cli_version": "gemini-cli@0.36.0",
"setup": {
"skills": [
{
"action": "link",
"path": "/path/to/my-skill"
}
]
}
}

monkeypatch.setenv("HOME", real_home)
with (
patch('generators.models.gemini_cli.os.makedirs'),
patch('generators.models.gemini_cli.open', create=True),
):
GeminiCliGenerator(config)

assert mock_run.call_count == 3
calls = [call[0][0] for call in mock_run.call_args_list]

expected_cmd = [
"npm",
"exec",
"--yes",
"gemini-cli@0.36.0",
"--",
"skills",
"link",
"/path/to/my-skill",
"--consent",
]
assert expected_cmd in calls


@patch('generators.models.gemini_cli.subprocess.run')
@patch('generators.models.gemini_cli.os.path.exists')
def test_setup_skill_dict_install_by_path(
mock_exists,
mock_run,
monkeypatch,
):
real_home = "/fake/real_home"
mock_exists.return_value = False
mock_run.return_value = MagicMock(returncode=0, stdout="success")

config = {
"gemini_cli_version": "gemini-cli@0.36.0",
"setup": {
"skills": [
{
"action": "install",
"path": "/path/to/my-skill"
}
]
}
}

monkeypatch.setenv("HOME", real_home)
with (
patch('generators.models.gemini_cli.os.makedirs'),
patch('generators.models.gemini_cli.open', create=True),
):
GeminiCliGenerator(config)

assert mock_run.call_count == 3
calls = [call[0][0] for call in mock_run.call_args_list]

expected_cmd = [
"npm",
"exec",
"--yes",
"gemini-cli@0.36.0",
"--",
"skills",
"install",
"/path/to/my-skill",
"--consent",
]
assert expected_cmd in calls


@patch('generators.models.gemini_cli.subprocess.run')
@patch('generators.models.gemini_cli.os.path.exists')
def test_setup_skill_dict_install_by_name(
mock_exists,
mock_run,
monkeypatch,
):
real_home = "/fake/real_home"
mock_exists.return_value = False
mock_run.return_value = MagicMock(returncode=0, stdout="success")

config = {
"gemini_cli_version": "gemini-cli@0.36.0",
"setup": {
"skills": [
{
"action": "install",
"name": "my-skill-package"
}
]
}
}

monkeypatch.setenv("HOME", real_home)
with (
patch('generators.models.gemini_cli.os.makedirs'),
patch('generators.models.gemini_cli.open', create=True),
):
GeminiCliGenerator(config)

assert mock_run.call_count == 3
calls = [call[0][0] for call in mock_run.call_args_list]

expected_cmd = [
"npm",
"exec",
"--yes",
"gemini-cli@0.36.0",
"--",
"skills",
"install",
"my-skill-package",
"--consent",
]
assert expected_cmd in calls


@patch('generators.models.gemini_cli.subprocess.run')
@patch('generators.models.gemini_cli.os.path.exists')
def test_setup_skill_dict_enable(
mock_exists,
mock_run,
monkeypatch,
):
real_home = "/fake/real_home"
mock_exists.return_value = False
mock_run.return_value = MagicMock(returncode=0, stdout="success")

config = {
"gemini_cli_version": "gemini-cli@0.36.0",
"setup": {
"skills": [
{
"action": "enable",
"name": "my-skill"
}
]
}
}

monkeypatch.setenv("HOME", real_home)
with (
patch('generators.models.gemini_cli.os.makedirs'),
patch('generators.models.gemini_cli.open', create=True),
):
GeminiCliGenerator(config)

assert mock_run.call_count == 3
calls = [call[0][0] for call in mock_run.call_args_list]

expected_cmd = [
"npm",
"exec",
"--yes",
"gemini-cli@0.36.0",
"--",
"skills",
"enable",
"my-skill",
]
assert expected_cmd in calls


@patch('generators.models.gemini_cli.subprocess.run')
@patch('generators.models.gemini_cli.os.path.exists')
def test_setup_skill_dict_disable(
mock_exists,
mock_run,
monkeypatch,
):
real_home = "/fake/real_home"
mock_exists.return_value = False
mock_run.return_value = MagicMock(returncode=0, stdout="success")

config = {
"gemini_cli_version": "gemini-cli@0.36.0",
"setup": {
"skills": [
{
"action": "disable",
"name": "my-skill"
}
]
}
}

monkeypatch.setenv("HOME", real_home)
with (
patch('generators.models.gemini_cli.os.makedirs'),
patch('generators.models.gemini_cli.open', create=True),
):
GeminiCliGenerator(config)

assert mock_run.call_count == 3
calls = [call[0][0] for call in mock_run.call_args_list]

expected_cmd = [
"npm",
"exec",
"--yes",
"gemini-cli@0.36.0",
"--",
"skills",
"disable",
"my-skill",
]
assert expected_cmd in calls
Loading