diff --git a/pyproject.toml b/pyproject.toml index 68392ea..7eedd77 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ [project] name = "postgresql-charms-single-kernel" description = "Shared and reusable code for PostgreSQL-related charms" -version = "16.1.12" +version = "16.2.1" readme = "README.md" license = {file = "LICENSE"} authors = [ @@ -22,6 +22,9 @@ dependencies = [ "tenacity>=9.0.0", ] +[project.optional-dependencies] +postgresql = ["httpx; python_version >= '3.12'"] + [build-system] requires = ["uv_build>=0.11.0,<0.12.0"] build-backend = "uv_build" diff --git a/single_kernel_postgresql/config/literals.py b/single_kernel_postgresql/config/literals.py index 385cf4a..a326f61 100644 --- a/single_kernel_postgresql/config/literals.py +++ b/single_kernel_postgresql/config/literals.py @@ -22,6 +22,8 @@ USER = "operator" SYSTEM_USERS = [MONITORING_USER, REPLICATION_USER, REWIND_USER, USER] +API_REQUEST_TIMEOUT = 5 + class Substrates(str, Enum): """Possible substrates.""" diff --git a/single_kernel_postgresql/utils/__init__.py b/single_kernel_postgresql/utils/__init__.py index aae77c5..f1e1f1e 100644 --- a/single_kernel_postgresql/utils/__init__.py +++ b/single_kernel_postgresql/utils/__init__.py @@ -1,3 +1,197 @@ -# Copyright 2025 Canonical Ltd. +# Copyright 2022 Canonical Ltd. # See LICENSE file for licensing details. -"""Utils and helpers for PostgreSQL charms.""" + +"""A collection of utility functions that are used in the charm.""" + +import os +import pwd +import re +import secrets +import string +from asyncio import as_completed, create_task, run, wait +from contextlib import suppress +from ssl import CERT_NONE, create_default_context +from typing import Any + +from httpx import AsyncClient, BasicAuth, HTTPError + +from ..config.literals import API_REQUEST_TIMEOUT, Substrates + + +def new_password() -> str: + """Generate a random password string. + + Returns: + A random password string. + """ + choices = string.ascii_letters + string.digits + password = "".join([secrets.choice(choices) for _ in range(16)]) + return password + + +def split_mem(mem_str) -> tuple: + """Split a memory string into a number and a unit. + + Args: + mem_str: a string representing a memory value, e.g. "1Gi" + """ + pattern = r"^(\d+)(\w+)$" + parts = re.match(pattern, mem_str) + if parts: + return parts.groups() + return None, "No unit found" + + +def any_memory_to_bytes(mem_str) -> int: + """Convert a memory string to bytes. + + Args: + mem_str: a string representing a memory value, e.g. "1Gi" + """ + units = { + "KI": 1024, + "K": 10**3, + "MI": 1048576, + "M": 10**6, + "GI": 1073741824, + "G": 10**9, + "TI": 1099511627776, + "T": 10**12, + } + try: + num = int(mem_str) + return num + except ValueError as e: + memory, unit = split_mem(mem_str) + unit = unit.upper() + if unit not in units: + raise ValueError(f"Invalid memory definition in '{mem_str}'") from e + + num = int(memory) + return int(num * units[unit]) + + +def any_cpu_to_cores(cpu_str) -> int: + """Convert a CPU string to cores. + + Args: + cpu_str: a string representing a CPU value, as integer or millis + """ + if cpu_str.endswith("m"): + # convert millis to cores, undercommited + return int(cpu_str[:-1]) // 1000 + return int(cpu_str) + + +def label2name(label: str) -> str: + """Convert a unit label (with `-`) to a unit name (with `/`). + + Args: + label: The label to convert. + + Returns: + The converted name. + """ + return "/".join(label.rsplit("-", 1)) + + +def render_file( + substrate: Substrates, path: str, content: str, mode: int, change_owner: bool = True +) -> None: + """Write a content rendered from a template to a file. + + Args: + substrate: Charm substrate. + path: the path to the file. + content: the data to be written to the file. + mode: access permission mask applied to the + file using chmod (e.g. 0o640). + change_owner: whether to change the file owner + to the _daemon_ user. + """ + # TODO: keep this method to use it also for generating replication configuration files and + # move it to an utils / helpers file. + # Write the content to the file. + with open(path, "w+") as file: + file.write(content) + # Ensure correct permissions are set on the file. + os.chmod(path, mode) + if change_owner: + _change_owner(substrate, path) + + +def create_directory(substrate: Substrates, path: str, mode: int) -> None: + """Creates a directory. + + Args: + substrate: Charm substrate. + path: the path of the directory that should be created. + mode: access permission mask applied to the + directory using chmod (e.g. 0o640). + """ + os.makedirs(path, mode=mode, exist_ok=True) + # Ensure correct permissions are set on the directory. + os.chmod(path, mode) + _change_owner(substrate, path) + + +def _change_owner(substrate: Substrates, path: str) -> None: + """Change the ownership of a file or a directory to the postgres user. + + Args: + substrate: Charm substrate. + path: path to a file or directory. + """ + try: + # Get the uid/gid for the _daemon_ user. + user_database = ( + pwd.getpwnam("_daemon_") if substrate == Substrates.VM else pwd.getpwnam("postgres") + ) + # Set the correct ownership for the file or directory. + os.chown(path, uid=user_database.pw_uid, gid=user_database.pw_gid) + except KeyError: + # Ignore non existing user error when it wasn't created yet. + pass + + +async def _httpx_get_request( + url: str, cafile: str, auth: BasicAuth | None = None, verify: bool = True +) -> dict[str, Any] | None: + ssl_ctx = create_default_context() + if verify: + with suppress(FileNotFoundError): + ssl_ctx.load_verify_locations(cafile=cafile) + else: + ssl_ctx.check_hostname = False + ssl_ctx.verify_mode = CERT_NONE + async with AsyncClient(auth=auth, timeout=API_REQUEST_TIMEOUT, verify=ssl_ctx) as client: + try: + return (await client.get(url)).raise_for_status().json() + except (HTTPError, ValueError): + return None + + +async def _async_get_request( + uri: str, endpoints: list[str], cafile: str, auth: BasicAuth | None, verify: bool = True +) -> dict[str, Any] | None: + tasks = [ + create_task(_httpx_get_request(f"https://{ip}:8008{uri}", cafile, auth, verify)) + for ip in endpoints + ] + for task in as_completed(tasks): + if result := await task: + for task in tasks: + task.cancel() + await wait(tasks) + return result + + +def parallel_patroni_get_request( + uri: str, + endpoints: list[str], + cafile: str, + auth: BasicAuth | None = None, + verify: bool = True, +) -> dict[str, Any] | None: + """Call all possible patroni endpoints in parallel.""" + return run(_async_get_request(uri, endpoints, cafile, auth, verify)) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..46a5f4f --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,107 @@ +# Copyright 2021 Canonical Ltd. +# See LICENSE file for licensing details. + +import re +from unittest.mock import mock_open, patch + +from single_kernel_postgresql.config.literals import Substrates +from single_kernel_postgresql.utils import ( + any_cpu_to_cores, + any_memory_to_bytes, + create_directory, + label2name, + new_password, + render_file, +) + + +def test_any_memory_to_bytes(): + assert any_memory_to_bytes(1024) == 1024 + + assert any_memory_to_bytes("1KI") == 1024 + + try: + any_memory_to_bytes("KI") + assert False + except ValueError as e: + assert str(e) == "Invalid memory definition in 'KI'" + + +def test_label2name(): + assert label2name("postgresql-k8s-1") == "postgresql-k8s/1" + + +def test_any_cpu_to_cores(): + assert any_cpu_to_cores("12") == 12 + assert any_cpu_to_cores("1000m") == 1 + + +def test_new_password(): + # Test the password generation twice in order to check if we get different passwords and + # that they meet the required criteria. + first_password = new_password() + assert len(first_password) == 16 + assert re.fullmatch("[a-zA-Z0-9\b]{16}$", first_password) is not None + + second_password = new_password() + assert re.fullmatch("[a-zA-Z0-9\b]{16}$", second_password) is not None + assert second_password != first_password + + +def test_render_file(): + with ( + patch("os.chmod") as _chmod, + patch("os.chown") as _chown, + patch("pwd.getpwnam") as _pwnam, + patch("tempfile.NamedTemporaryFile") as _temp_file, + ): + # Set a mocked temporary filename. + filename = "/tmp/temporaryfilename" + _temp_file.return_value.name = filename + # Setup a mock for the `open` method. + mock = mock_open() + # Patch the `open` method with our mock. + with patch("builtins.open", mock, create=True): + # Set the uid/gid return values for lookup of 'postgres' user. + _pwnam.return_value.pw_uid = 35 + _pwnam.return_value.pw_gid = 35 + # Call the method using a temporary configuration file. + render_file(Substrates.VM, filename, "rendered-content", 0o640) + + # Check the rendered file is opened with "w+" mode. + assert mock.call_args_list[0][0] == (filename, "w+") + # Ensure that the correct user is lookup up. + _pwnam.assert_called_with("_daemon_") + # Ensure the file is chmod'd correctly. + _chmod.assert_called_with(filename, 0o640) + # Ensure the file is chown'd correctly. + _chown.assert_called_with(filename, uid=35, gid=35) + + # Test when it's requested to not change the file owner. + mock.reset_mock() + _pwnam.reset_mock() + _chmod.reset_mock() + _chown.reset_mock() + with patch("builtins.open", mock, create=True): + render_file(Substrates.VM, filename, "rendered-content", 0o640, change_owner=False) + _pwnam.assert_not_called() + _chmod.assert_called_once_with(filename, 0o640) + _chown.assert_not_called() + + +def test_create_directory(): + with ( + patch("os.chmod") as _chmod, + patch("os.chown") as _chown, + patch("os.makedirs") as _makedirs, + patch("pwd.getpwnam") as _pwnam, + ): + _pwnam.return_value.pw_uid = 35 + _pwnam.return_value.pw_gid = 35 + + create_directory(Substrates.K8S, "test", 0o640) + + _makedirs.assert_called_once_with("test", mode=0o640, exist_ok=True) + _chmod.assert_called_once_with("test", 0o640) + _chown.assert_called_once_with("test", uid=35, gid=35) + _pwnam.assert_called_with("postgres") diff --git a/tox.ini b/tox.ini index c2748e0..c1ae8ef 100644 --- a/tox.ini +++ b/tox.ini @@ -22,7 +22,7 @@ allowlist_externals = [testenv:format] description = Apply coding style standards to code commands_pre = - uv sync --active --group format + uv sync --active --group format --all-extras commands = uv run --active ruff check --fix {[vars]all_path} uv run --active ruff format {[vars]all_path} @@ -31,7 +31,7 @@ commands = [testenv:lint] description = Check code against coding style standards commands_pre = - uv sync --active --group lint --group format + uv sync --active --group lint --group format --all-extras commands = uv lock --check uv run --active codespell "{tox_root}" --skip "{tox_root}/.git" --skip "{tox_root}/.tox" \ @@ -45,7 +45,7 @@ commands = [testenv:unit] description = Run unit tests commands_pre = - uv sync --active --group unit + uv sync --active --group unit --all-extras commands = uv run --active coverage run --source={[vars]src_path} \ -m pytest -v --tb native -s {posargs} {[vars]tests_path}/unit diff --git a/uv.lock b/uv.lock index 64f8710..e135cf7 100644 --- a/uv.lock +++ b/uv.lock @@ -8,6 +8,28 @@ resolution-markers = [ "python_full_version < '3.9'", ] +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna", marker = "python_full_version >= '3.12'" }, + { name = "typing-extensions", marker = "python_full_version == '3.12.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622, upload-time = "2026-03-24T12:59:09.671Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + [[package]] name = "codespell" version = "2.4.2" @@ -151,6 +173,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, ] +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi", marker = "python_full_version >= '3.12'" }, + { name = "h11", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "python_full_version >= '3.12'" }, + { name = "certifi", marker = "python_full_version >= '3.12'" }, + { name = "httpcore", marker = "python_full_version >= '3.12'" }, + { name = "idna", marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/88/bcf9709822fe69d02c2a6a77956c98ce6ea8ca8767a9aadcedc7eb6a2390/idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d", size = 203770, upload-time = "2026-05-22T00:16:18.781Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/16/70255075a9859a0e3adb789b68ceb0e210dec03934245fd98d248226572f/idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", size = 74165, upload-time = "2026-05-22T00:16:16.698Z" }, +] + [[package]] name = "importlib-metadata" version = "8.5.0" @@ -287,7 +355,7 @@ wheels = [ [[package]] name = "postgresql-charms-single-kernel" -version = "16.1.12" +version = "16.2.1" source = { editable = "." } dependencies = [ { name = "ops", version = "2.23.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, @@ -299,6 +367,11 @@ dependencies = [ { name = "tenacity", version = "9.1.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] +[package.optional-dependencies] +postgresql = [ + { name = "httpx", marker = "python_full_version >= '3.12'" }, +] + [package.dev-dependencies] format = [ { name = "ruff", marker = "python_full_version >= '3.12'" }, @@ -314,10 +387,12 @@ unit = [ [package.metadata] requires-dist = [ + { name = "httpx", marker = "python_full_version >= '3.12' and extra == 'postgresql'" }, { name = "ops", specifier = ">=2.0.0" }, { name = "psycopg2", specifier = ">=2.9.10" }, { name = "tenacity", specifier = ">=9.0.0" }, ] +provides-extras = ["postgresql"] [package.metadata.requires-dev] format = [{ name = "ruff", marker = "python_full_version >= '3.12'", specifier = "==0.15.13" }]