From 633da99962866dbefa223e071bca427ad534fce4 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 12:53:53 +0100 Subject: [PATCH 01/12] Adds sync API and refactors core logic for async/sync parity Introduces a synchronous job queue API alongside the existing async API, enabling seamless interoperability between both interfaces. Refactors core input validation, Lua script loading, and result formatting into a shared core module to ensure consistent behavior and error handling. Updates documentation and adds comprehensive sync API tests to ensure feature parity and cross-API compatibility. --- README.md | 67 +++++++- src/fq/core.py | 289 ++++++++++++++++++++++++++++++++ src/fq/queue.py | 294 +++++--------------------------- src/fq/sync/__init__.py | 4 + src/fq/sync/queue.py | 334 +++++++++++++++++++++++++++++++++++++ tests/test_sync_queue.py | 352 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 1087 insertions(+), 253 deletions(-) create mode 100644 src/fq/core.py create mode 100644 src/fq/sync/__init__.py create mode 100644 src/fq/sync/queue.py create mode 100644 tests/test_sync_queue.py diff --git a/README.md b/README.md index e1fe8e9..f8675c1 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,13 @@ Flowdacity Queue ================ -Flowdacity Queue (FQ) is an asyncio-friendly, rate-limited job queue built on Redis. It stores jobs per queue type and queue id, enforces per-queue dequeue intervals, automatically requeues expired jobs, and exposes metrics to understand throughput and queue depth. +Flowdacity Queue (FQ) is a rate-limited job queue built on Redis with supported async and sync APIs. It stores jobs per queue type and queue id, enforces per-queue dequeue intervals, automatically requeues expired jobs, and exposes metrics to understand throughput and queue depth. ## Features - Per-queue rate limiting using millisecond intervals. -- Async Redis client with Lua scripts for predictable behavior. +- Async and sync APIs backed by Redis clients from redis-py. +- Lua scripts for predictable queue behavior across both APIs. - Automatic retries with configurable limits (including infinite retries). - Metrics for enqueue/dequeue counts and queue lengths. - Works with TCP or Unix socket Redis deployments and supports Redis Cluster. @@ -60,7 +61,13 @@ config = { > unixsocketperm 755 > ``` -## Quickstart +## Async API + +The top-level namespace is async-first: + +```python +from fq import FQ +``` ```python import asyncio @@ -114,13 +121,65 @@ async def main(): asyncio.run(main()) ``` -Common operations: +## Sync API + +The sync API is available from the `fq.sync` namespace and supports the same queue operations without `await`: + +```python +import uuid +from fq.sync import FQ + + +config = { + "fq": { + "job_expire_interval": 5000, + "job_requeue_interval": 5000, + "default_job_requeue_limit": -1, + }, + "redis": { + "db": 0, + "key_prefix": "queue_server", + "conn_type": "tcp_sock", + "host": "127.0.0.1", + "port": 6379, + "password": "", + "clustered": False, + "unix_socket_path": "/tmp/redis.sock", + }, +} + +fq = FQ(config) +fq.initialize() + +job_id = str(uuid.uuid4()) +fq.enqueue( + payload={"message": "hello, world"}, + interval=1000, + job_id=job_id, + queue_id="user001", + queue_type="sms", +) + +job = fq.dequeue(queue_type="sms") +if job["status"] == "success": + fq.finish( + queue_type="sms", + queue_id=job["queue_id"], + job_id=job["job_id"], + ) + +fq.close() +``` + +## Common Operations - `await fq.requeue()` — move expired jobs back onto their queues. - `await fq.interval(interval=5000, queue_id="user001", queue_type="sms")` — change a queue’s rate limit on the fly. - `await fq.metrics()` — global metrics; pass `queue_type` and/or `queue_id` for scoped stats and queue length. - `await fq.clear_queue(queue_type="sms", queue_id="user001", purge_all=True)` — drop queued jobs and their payload/interval metadata. +For sync code, use the same method names without `await`: `fq.requeue()`, `fq.interval(...)`, `fq.metrics()`, and `fq.clear_queue(...)`. + ## Development - Start Redis for local development: `make redis-up` (binds to `localhost:6379`). diff --git a/src/fq/core.py b/src/fq/core.py new file mode 100644 index 0000000..afaaeb0 --- /dev/null +++ b/src/fq/core.py @@ -0,0 +1,289 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +import os +from collections.abc import Mapping + +from fq.exceptions import BadArgumentException, FQException +from fq.utils import ( + convert_to_str, + deserialize_payload, + generate_epoch, + is_valid_identifier, + is_valid_interval, + is_valid_requeue_limit, + serialize_payload, +) + +LUA_SCRIPT_NAMES = ("enqueue", "dequeue", "finish", "interval", "requeue", "metrics") + + +def normalize_config(config): + if not isinstance(config, Mapping): + raise FQException("Config must be a mapping with redis and fq sections") + + normalized = {} + for section_name, section_values in config.items(): + if not isinstance(section_values, Mapping): + raise FQException("Config section '%s' must be a mapping" % section_name) + + normalized[str(section_name)] = { + str(option): value for option, value in section_values.items() + } + + if "redis" not in normalized or "fq" not in normalized: + raise FQException("Config missing required sections: redis, fq") + + redis_config = normalized["redis"] + fq_config = normalized["fq"] + + if "key_prefix" not in redis_config: + raise FQException("Missing config: redis.key_prefix") + if not isinstance(redis_config["key_prefix"], str) or not redis_config[ + "key_prefix" + ]: + raise FQException("Invalid config: redis.key_prefix must be a non-empty string") + + if "conn_type" not in redis_config: + raise FQException("Missing config: redis.conn_type") + if redis_config["conn_type"] not in {"tcp_sock", "unix_sock"}: + raise FQException( + "Invalid config: redis.conn_type must be 'tcp_sock' or 'unix_sock'" + ) + + if "db" not in redis_config: + raise FQException("Missing config: redis.db") + if isinstance(redis_config["db"], bool) or not isinstance(redis_config["db"], int): + raise FQException("Invalid config: redis.db must be an integer") + + if "job_expire_interval" not in fq_config: + raise FQException("Missing config: fq.job_expire_interval") + if not is_valid_interval(fq_config["job_expire_interval"]): + raise FQException( + "Invalid config: fq.job_expire_interval must be a positive integer" + ) + + if "job_requeue_interval" not in fq_config: + raise FQException("Missing config: fq.job_requeue_interval") + if not is_valid_interval(fq_config["job_requeue_interval"]): + raise FQException( + "Invalid config: fq.job_requeue_interval must be a positive integer" + ) + + if "default_job_requeue_limit" not in fq_config: + raise FQException("Missing config: fq.default_job_requeue_limit") + if not is_valid_requeue_limit(fq_config["default_job_requeue_limit"]): + raise FQException( + "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" + ) + + if redis_config["conn_type"] == "unix_sock": + if "unix_socket_path" not in redis_config: + raise FQException("Missing config: redis.unix_socket_path") + if not isinstance(redis_config["unix_socket_path"], str) or not redis_config[ + "unix_socket_path" + ]: + raise FQException( + "Invalid config: redis.unix_socket_path must be a non-empty string" + ) + + if redis_config["conn_type"] == "tcp_sock": + if "host" not in redis_config: + raise FQException("Missing config: redis.host") + if not isinstance(redis_config["host"], str) or not redis_config["host"]: + raise FQException("Invalid config: redis.host must be a non-empty string") + + if "port" not in redis_config: + raise FQException("Missing config: redis.port") + if isinstance(redis_config["port"], bool) or not isinstance( + redis_config["port"], int + ): + raise FQException("Invalid config: redis.port must be an integer") + + if "clustered" in redis_config and not isinstance( + redis_config["clustered"], bool + ): + raise FQException("Invalid config: redis.clustered must be a boolean") + + if "password" in redis_config and redis_config["password"] is not None: + if not isinstance(redis_config["password"], str): + raise FQException("Invalid config: redis.password must be a string") + + return normalized + + +def load_lua_scripts(instance, redis_client): + lua_script_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "scripts", "lua" + ) + + for script_name in LUA_SCRIPT_NAMES: + with open( + os.path.join(lua_script_path, "%s.lua" % script_name), + "r", + encoding="utf-8", + ) as script_file: + script = script_file.read() + setattr(instance, "_lua_%s_script" % script_name, script) + setattr( + instance, + "_lua_%s" % script_name, + redis_client.register_script(script), + ) + + +def validate_enqueue_arguments( + payload, + interval, + job_id, + queue_id, + queue_type, + requeue_limit, + default_requeue_limit, +): + if not is_valid_interval(interval): + raise BadArgumentException("`interval` has an invalid value.") + + if not is_valid_identifier(job_id): + raise BadArgumentException("`job_id` has an invalid value.") + + if not is_valid_identifier(queue_id): + raise BadArgumentException("`queue_id` has an invalid value.") + + if not is_valid_identifier(queue_type): + raise BadArgumentException("`queue_type` has an invalid value.") + + if requeue_limit is None: + requeue_limit = default_requeue_limit + + if not is_valid_requeue_limit(requeue_limit): + raise BadArgumentException("`requeue_limit` has an invalid value.") + + try: + serialized_payload = serialize_payload(payload) + except TypeError: + raise BadArgumentException("can not serialize.") + + return serialized_payload, requeue_limit + + +def validate_dequeue_arguments(queue_type): + if not is_valid_identifier(queue_type): + raise BadArgumentException("`queue_type` has an invalid value.") + + +def validate_finish_arguments(job_id, queue_id, queue_type): + if not is_valid_identifier(job_id): + raise BadArgumentException("`job_id` has an invalid value.") + + if not is_valid_identifier(queue_id): + raise BadArgumentException("`queue_id` has an invalid value.") + + if not is_valid_identifier(queue_type): + raise BadArgumentException("`queue_type` has an invalid value.") + + +def validate_interval_arguments(interval, queue_id, queue_type): + if not is_valid_interval(interval): + raise BadArgumentException("`interval` has an invalid value.") + + if not is_valid_identifier(queue_id): + raise BadArgumentException("`queue_id` has an invalid value.") + + if not is_valid_identifier(queue_type): + raise BadArgumentException("`queue_type` has an invalid value.") + + +def validate_metrics_arguments(queue_type, queue_id): + if queue_id is not None and not is_valid_identifier(queue_id): + raise BadArgumentException("`queue_id` has an invalid value.") + + if queue_type is not None and not is_valid_identifier(queue_type): + raise BadArgumentException("`queue_type` has an invalid value.") + + +def validate_clear_queue_arguments(queue_type, queue_id): + if queue_id is None or not is_valid_identifier(queue_id): + raise BadArgumentException("`queue_id` has an invalid value.") + + if queue_type is None or not is_valid_identifier(queue_type): + raise BadArgumentException("`queue_type` has an invalid value.") + + +def validate_get_queue_length_arguments(queue_type, queue_id): + if not is_valid_identifier(queue_type): + raise BadArgumentException("`queue_type` has an invalid value.") + + if not is_valid_identifier(queue_id): + raise BadArgumentException("`queue_id` has an invalid value.") + + +def decode_redis_value(value): + if isinstance(value, bytes): + return value.decode("utf-8") + return value + + +def format_dequeue_response(dequeue_response): + if len(dequeue_response) < 4: + return {"status": "failure"} + + queue_id, job_id, payload, requeues_remaining = dequeue_response + + if payload is None: + return {"status": "failure"} + + payload = deserialize_payload(payload) + + return { + "status": "success", + "queue_id": decode_redis_value(queue_id), + "job_id": decode_redis_value(job_id), + "payload": payload, + "requeues_remaining": int(requeues_remaining), + } + + +def format_metrics_counts(enqueue_details, dequeue_details): + enqueue_counts = {} + dequeue_counts = {} + for i in range(0, len(enqueue_details), 2): + enqueue_counts[str(decode_redis_value(enqueue_details[i]))] = int( + enqueue_details[i + 1] or 0 + ) + dequeue_counts[str(decode_redis_value(dequeue_details[i]))] = int( + dequeue_details[i + 1] or 0 + ) + return enqueue_counts, dequeue_counts + + +def format_queue_types(active_queue_types, ready_queue_types): + return convert_to_str(active_queue_types | ready_queue_types) + + +def format_queue_ids(ready_queues, active_queues): + active_queues = [decode_redis_value(i).split(":")[0] for i in active_queues] + all_queue_set = set(ready_queues) | set(active_queues) + return convert_to_str(all_queue_set) + + +def enqueue_script_args( + key_prefix, + queue_type, + queue_id, + job_id, + serialized_payload, + interval, + requeue_limit, +): + timestamp = str(generate_epoch()) + keys = [key_prefix, queue_type] + args = [ + timestamp, + queue_id, + job_id, + serialized_payload, + interval, + requeue_limit, + ] + return keys, args diff --git a/src/fq/queue.py b/src/fq/queue.py index cde545c..52b2658 100644 --- a/src/fq/queue.py +++ b/src/fq/queue.py @@ -3,20 +3,26 @@ # Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. import asyncio -import os -from collections.abc import Mapping from redis.asyncio import Redis from redis.asyncio.cluster import RedisCluster -from fq.utils import ( - is_valid_identifier, - is_valid_interval, - is_valid_requeue_limit, +from fq.core import ( + enqueue_script_args, + format_dequeue_response, + format_metrics_counts, + format_queue_ids, + format_queue_types, generate_epoch, - serialize_payload, - deserialize_payload, - convert_to_str, + load_lua_scripts, + normalize_config, + validate_clear_queue_arguments, + validate_dequeue_arguments, + validate_enqueue_arguments, + validate_finish_arguments, + validate_get_queue_length_arguments, + validate_interval_arguments, + validate_metrics_arguments, ) from fq.exceptions import FQException, BadArgumentException @@ -36,105 +42,7 @@ def __init__(self, config): 2. Validate the config shape. """ self._r = None # redis client placeholder - if not isinstance(config, Mapping): - raise FQException("Config must be a mapping with redis and fq sections") - - normalized = {} - for section_name, section_values in config.items(): - if not isinstance(section_values, Mapping): - raise FQException( - "Config section '%s' must be a mapping" % section_name - ) - - normalized[str(section_name)] = { - str(option): value for option, value in section_values.items() - } - - if "redis" not in normalized or "fq" not in normalized: - raise FQException("Config missing required sections: redis, fq") - - redis_config = normalized["redis"] - fq_config = normalized["fq"] - - if "key_prefix" not in redis_config: - raise FQException("Missing config: redis.key_prefix") - if not isinstance(redis_config["key_prefix"], str) or not redis_config[ - "key_prefix" - ]: - raise FQException( - "Invalid config: redis.key_prefix must be a non-empty string" - ) - - if "conn_type" not in redis_config: - raise FQException("Missing config: redis.conn_type") - if redis_config["conn_type"] not in {"tcp_sock", "unix_sock"}: - raise FQException( - "Invalid config: redis.conn_type must be 'tcp_sock' or 'unix_sock'" - ) - - if "db" not in redis_config: - raise FQException("Missing config: redis.db") - if isinstance(redis_config["db"], bool) or not isinstance( - redis_config["db"], int - ): - raise FQException("Invalid config: redis.db must be an integer") - - if "job_expire_interval" not in fq_config: - raise FQException("Missing config: fq.job_expire_interval") - if not is_valid_interval(fq_config["job_expire_interval"]): - raise FQException( - "Invalid config: fq.job_expire_interval must be a positive integer" - ) - - if "job_requeue_interval" not in fq_config: - raise FQException("Missing config: fq.job_requeue_interval") - if not is_valid_interval(fq_config["job_requeue_interval"]): - raise FQException( - "Invalid config: fq.job_requeue_interval must be a positive integer" - ) - - if "default_job_requeue_limit" not in fq_config: - raise FQException("Missing config: fq.default_job_requeue_limit") - if not is_valid_requeue_limit(fq_config["default_job_requeue_limit"]): - raise FQException( - "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" - ) - - if redis_config["conn_type"] == "unix_sock": - if "unix_socket_path" not in redis_config: - raise FQException("Missing config: redis.unix_socket_path") - if not isinstance(redis_config["unix_socket_path"], str) or not redis_config[ - "unix_socket_path" - ]: - raise FQException( - "Invalid config: redis.unix_socket_path must be a non-empty string" - ) - - if redis_config["conn_type"] == "tcp_sock": - if "host" not in redis_config: - raise FQException("Missing config: redis.host") - if not isinstance(redis_config["host"], str) or not redis_config["host"]: - raise FQException( - "Invalid config: redis.host must be a non-empty string" - ) - - if "port" not in redis_config: - raise FQException("Missing config: redis.port") - if isinstance(redis_config["port"], bool) or not isinstance( - redis_config["port"], int - ): - raise FQException("Invalid config: redis.port must be an integer") - - if "clustered" in redis_config and not isinstance( - redis_config["clustered"], bool - ): - raise FQException("Invalid config: redis.clustered must be a boolean") - - if "password" in redis_config and redis_config["password"] is not None: - if not isinstance(redis_config["password"], str): - raise FQException("Invalid config: redis.password must be a string") - - self.config = normalized + self.config = normalize_config(config) async def initialize(self): """Async initializer to set up redis and lua scripts.""" @@ -205,33 +113,7 @@ def redis_client(self): def _load_lua_scripts(self): """Loads all lua scripts required by FQ.""" - # load lua scripts - lua_script_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "scripts/lua" - ) - with open(os.path.join(lua_script_path, "enqueue.lua"), "r") as enqueue_file: - self._lua_enqueue_script = enqueue_file.read() - self._lua_enqueue = self._r.register_script(self._lua_enqueue_script) - - with open(os.path.join(lua_script_path, "dequeue.lua"), "r") as dequeue_file: - self._lua_dequeue_script = dequeue_file.read() - self._lua_dequeue = self._r.register_script(self._lua_dequeue_script) - - with open(os.path.join(lua_script_path, "finish.lua"), "r") as finish_file: - self._lua_finish_script = finish_file.read() - self._lua_finish = self._r.register_script(self._lua_finish_script) - - with open(os.path.join(lua_script_path, "interval.lua"), "r") as interval_file: - self._lua_interval_script = interval_file.read() - self._lua_interval = self._r.register_script(self._lua_interval_script) - - with open(os.path.join(lua_script_path, "requeue.lua"), "r") as requeue_file: - self._lua_requeue_script = requeue_file.read() - self._lua_requeue = self._r.register_script(self._lua_requeue_script) - - with open(os.path.join(lua_script_path, "metrics.lua"), "r") as metrics_file: - self._lua_metrics_script = metrics_file.read() - self._lua_metrics = self._r.register_script(self._lua_metrics_script) + load_lua_scripts(self, self._r) def reload_lua_scripts(self): """Lets user reload the lua scripts in run time.""" @@ -249,40 +131,24 @@ async def enqueue( """Enqueues the job into the specified queue_id of a particular queue_type """ - # validate all the input - if not is_valid_interval(interval): - raise BadArgumentException("`interval` has an invalid value.") - - if not is_valid_identifier(job_id): - raise BadArgumentException("`job_id` has an invalid value.") - - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") - - if requeue_limit is None: - requeue_limit = self._default_job_requeue_limit - - if not is_valid_requeue_limit(requeue_limit): - raise BadArgumentException("`requeue_limit` has an invalid value.") - - try: - serialized_payload = serialize_payload(payload) - except TypeError as e: - raise BadArgumentException("can not serialize.") - - timestamp = str(generate_epoch()) - keys = [self._key_prefix, queue_type] - args = [ - timestamp, + serialized_payload, requeue_limit = validate_enqueue_arguments( + payload, + interval, + job_id, + queue_id, + queue_type, + requeue_limit, + self._default_job_requeue_limit, + ) + keys, args = enqueue_script_args( + self._key_prefix, + queue_type, queue_id, job_id, serialized_payload, interval, requeue_limit, - ] + ) await self._lua_enqueue(keys=keys, args=args) return {"status": "queued"} @@ -291,8 +157,7 @@ async def dequeue(self, queue_type="default"): based on the queue_type. If no job is ready, returns a failure status. """ - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + validate_dequeue_arguments(queue_type) timestamp = str(generate_epoch()) @@ -300,38 +165,14 @@ async def dequeue(self, queue_type="default"): args = [timestamp, self._job_expire_interval] dequeue_response = await self._lua_dequeue(keys=keys, args=args) - - if len(dequeue_response) < 4: - return {"status": "failure"} - - queue_id, job_id, payload, requeues_remaining = dequeue_response - - if payload is None: - return {"status": "failure"} - - payload = deserialize_payload(payload) - - return { - "status": "success", - "queue_id": queue_id.decode("utf-8"), - "job_id": job_id.decode("utf-8"), - "payload": payload, - "requeues_remaining": int(requeues_remaining), - } + return format_dequeue_response(dequeue_response) async def finish(self, job_id, queue_id, queue_type="default"): """Marks any dequeued job as *completed successfully*. Any job which gets a finish will be treated as complete and will be removed from the FQ. """ - if not is_valid_identifier(job_id): - raise BadArgumentException("`job_id` has an invalid value.") - - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + validate_finish_arguments(job_id, queue_id, queue_type) keys = [self._key_prefix, queue_type] @@ -348,15 +189,7 @@ async def interval(self, interval, queue_id, queue_type="default"): """Updates the interval for a specific queue_id of a particular queue type. """ - # validate all the input - if not is_valid_interval(interval): - raise BadArgumentException("`interval` has an invalid value.") - - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + validate_interval_arguments(interval, queue_id, queue_type) # generate the interval key interval_hmap_key = "%s:interval" % self._key_prefix @@ -410,11 +243,7 @@ async def metrics(self, queue_type=None, queue_id=None): * queue length of each queue. * list of queue ids for each queue type. """ - if queue_id is not None and not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if queue_type is not None and not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + validate_metrics_arguments(queue_type, queue_id) response = {"status": "failure"} if not queue_type and not queue_id: @@ -426,8 +255,7 @@ async def metrics(self, queue_type=None, queue_id=None): ready_queue_types = await self._r.smembers( "%s:ready:queue_type" % self._key_prefix ) - all_queue_types = active_queue_types | ready_queue_types - queue_types = convert_to_str(all_queue_types) + queue_types = format_queue_types(active_queue_types, ready_queue_types) # global rates for past 10 minutes timestamp = str(generate_epoch()) keys = [self._key_prefix] @@ -435,16 +263,9 @@ async def metrics(self, queue_type=None, queue_id=None): enqueue_details, dequeue_details = await self._lua_metrics( keys=keys, args=args ) - enqueue_counts = {} - dequeue_counts = {} - # the length of enqueue & dequeue details are always same. - for i in range(0, len(enqueue_details), 2): - enqueue_counts[str(enqueue_details[i])] = int( - enqueue_details[i + 1] or 0 - ) - dequeue_counts[str(dequeue_details[i])] = int( - dequeue_details[i + 1] or 0 - ) + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, dequeue_details + ) response.update( { @@ -463,20 +284,11 @@ async def metrics(self, queue_type=None, queue_id=None): pipe.zrange("%s:%s:active" % (self._key_prefix, queue_type), 0, -1) ready_queues, active_queues = await pipe.execute() # extract the queue_ids from the queue_id:job_id string - active_queues = [i.decode("utf-8").split(":")[0] for i in active_queues] - all_queue_set = set(ready_queues) | set(active_queues) - queue_list = convert_to_str(all_queue_set) + queue_list = format_queue_ids(ready_queues, active_queues) response.update({"status": "success", "queue_ids": queue_list}) return response elif queue_type and queue_id: # return specific details. - active_queue_types = await self._r.smembers( - "%s:active:queue_type" % self._key_prefix - ) - ready_queue_types = await self._r.smembers( - "%s:ready:queue_type" % self._key_prefix - ) - all_queue_types = active_queue_types | ready_queue_types # queue specific rates for past 10 minutes timestamp = str(generate_epoch()) keys = ["%s:%s:%s" % (self._key_prefix, queue_type, queue_id)] @@ -485,16 +297,9 @@ async def metrics(self, queue_type=None, queue_id=None): keys=keys, args=args ) - enqueue_counts = {} - dequeue_counts = {} - # the length of enqueue & dequeue details are always same. - for i in range(0, len(enqueue_details), 2): - enqueue_counts[str(enqueue_details[i])] = int( - enqueue_details[i + 1] or 0 - ) - dequeue_counts[str(dequeue_details[i])] = int( - dequeue_details[i + 1] or 0 - ) + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, dequeue_details + ) # get the queue length for the job queue queue_length = await self._r.llen( @@ -532,11 +337,7 @@ async def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): purge_all : if True, then it will remove the related resources from the redis. """ - if queue_id is None or not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if queue_type is None or not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + validate_clear_queue_arguments(queue_type, queue_id) response = {"status": "Failure", "message": "No queued calls found"} # remove from the primary sorted set @@ -592,12 +393,7 @@ async def get_queue_length(self, queue_type, queue_id): Redis key structure : key_prefix : queue_type : queue_id """ - # validate all the input - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") - - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") + validate_get_queue_length_arguments(queue_type, queue_id) redis_key = self._key_prefix + ":" + queue_type + ":" + queue_id current_queue_length = await self._r.llen(redis_key) diff --git a/src/fq/sync/__init__.py b/src/fq/sync/__init__.py new file mode 100644 index 0000000..990d355 --- /dev/null +++ b/src/fq/sync/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. +from .queue import FQ + +__all__ = ["FQ"] diff --git a/src/fq/sync/queue.py b/src/fq/sync/queue.py new file mode 100644 index 0000000..c8444e4 --- /dev/null +++ b/src/fq/sync/queue.py @@ -0,0 +1,334 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from redis import Redis, RedisCluster + +from fq.core import ( + decode_redis_value, + enqueue_script_args, + format_dequeue_response, + format_metrics_counts, + format_queue_ids, + format_queue_types, + generate_epoch, + load_lua_scripts, + normalize_config, + validate_clear_queue_arguments, + validate_dequeue_arguments, + validate_enqueue_arguments, + validate_finish_arguments, + validate_get_queue_length_arguments, + validate_interval_arguments, + validate_metrics_arguments, +) +from fq.exceptions import BadArgumentException, FQException + + +class FQ(object): + """Synchronous FQ API backed by redis-py's synchronous client.""" + + def __init__(self, config): + self._r = None + self.config = normalize_config(config) + + def initialize(self): + """Set up the synchronous Redis client and Lua scripts.""" + fq_config = self.config["fq"] + redis_config = self.config["redis"] + + self._key_prefix = redis_config["key_prefix"] + self._job_expire_interval = int(fq_config["job_expire_interval"]) + self._default_job_requeue_limit = int(fq_config["default_job_requeue_limit"]) + + redis_connection_type = redis_config["conn_type"] + db = redis_config["db"] + + if redis_connection_type == "unix_sock": + self._r = Redis( + db=db, + unix_socket_path=redis_config["unix_socket_path"], + ) + elif redis_connection_type == "tcp_sock": + isclustered = False + if "clustered" in redis_config: + isclustered = redis_config["clustered"] + + if isclustered: + self._r = RedisCluster( + host=redis_config["host"], + port=int(redis_config["port"]), + decode_responses=False, + socket_timeout=5, + ) + else: + self._r = Redis( + db=db, + host=redis_config["host"], + port=int(redis_config["port"]), + password=redis_config.get("password"), + ) + else: + raise FQException("Unknown redis conn_type: %s" % redis_connection_type) + + self._validate_redis_connection() + self._load_lua_scripts() + + def _validate_redis_connection(self): + """Ping Redis once to surface bad connection details early.""" + if self._r is None: + raise FQException("Redis client is not initialized") + + ping = getattr(self._r, "ping", None) + if not callable(ping): + return + + try: + result = ping() + except Exception as exc: + raise FQException("Failed to connect to Redis: %s" % exc) from exc + + if result is False: + raise FQException("Failed to connect to Redis: ping returned False") + + def redis_client(self): + return self._r + + def _load_lua_scripts(self): + """Loads all Lua scripts required by FQ.""" + load_lua_scripts(self, self._r) + + def reload_lua_scripts(self): + """Lets user reload the Lua scripts at run time.""" + self._load_lua_scripts() + + def enqueue( + self, + payload, + interval, + job_id, + queue_id, + queue_type="default", + requeue_limit=None, + ): + """Enqueue a job into the specified queue_id and queue_type.""" + serialized_payload, requeue_limit = validate_enqueue_arguments( + payload, + interval, + job_id, + queue_id, + queue_type, + requeue_limit, + self._default_job_requeue_limit, + ) + keys, args = enqueue_script_args( + self._key_prefix, + queue_type, + queue_id, + job_id, + serialized_payload, + interval, + requeue_limit, + ) + self._lua_enqueue(keys=keys, args=args) + return {"status": "queued"} + + def dequeue(self, queue_type="default"): + """Dequeue a ready job for the queue_type, or return failure.""" + validate_dequeue_arguments(queue_type) + + timestamp = str(generate_epoch()) + keys = [self._key_prefix, queue_type] + args = [timestamp, self._job_expire_interval] + + dequeue_response = self._lua_dequeue(keys=keys, args=args) + return format_dequeue_response(dequeue_response) + + def finish(self, job_id, queue_id, queue_type="default"): + """Mark a dequeued job as completed successfully.""" + validate_finish_arguments(job_id, queue_id, queue_type) + + keys = [self._key_prefix, queue_type] + args = [queue_id, job_id] + + finish_response = self._lua_finish(keys=keys, args=args) + if finish_response == 0: + return {"status": "failure"} + + return {"status": "success"} + + def interval(self, interval, queue_id, queue_type="default"): + """Update the interval for a queue_id and queue_type.""" + validate_interval_arguments(interval, queue_id, queue_type) + + interval_hmap_key = "%s:interval" % self._key_prefix + interval_queue_key = "%s:%s" % (queue_type, queue_id) + keys = [interval_hmap_key, interval_queue_key] + args = [interval] + + interval_response = self._lua_interval(keys=keys, args=args) + if interval_response == 0: + return {"status": "failure"} + return {"status": "success"} + + def requeue(self): + """Re-queue expired active jobs back into their ready queues.""" + timestamp = str(generate_epoch()) + active_queue_type_list = self._r.smembers( + "%s:active:queue_type" % self._key_prefix + ) + for queue_type in active_queue_type_list: + queue_type = decode_redis_value(queue_type) + keys = [self._key_prefix, queue_type] + args = [timestamp] + job_discard_list = self._lua_requeue(keys=keys, args=args) + for job in job_discard_list: + queue_id, job_id = decode_redis_value(job).split(":") + self.finish(job_id=job_id, queue_id=queue_id, queue_type=queue_type) + + def metrics(self, queue_type=None, queue_id=None): + """Return global, queue-type, or queue-specific metrics.""" + validate_metrics_arguments(queue_type, queue_id) + + response = {"status": "failure"} + if not queue_type and not queue_id: + active_queue_types = self._r.smembers( + "%s:active:queue_type" % self._key_prefix + ) + ready_queue_types = self._r.smembers( + "%s:ready:queue_type" % self._key_prefix + ) + queue_types = format_queue_types(active_queue_types, ready_queue_types) + + timestamp = str(generate_epoch()) + keys = [self._key_prefix] + args = [timestamp] + enqueue_details, dequeue_details = self._lua_metrics(keys=keys, args=args) + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, dequeue_details + ) + + response.update( + { + "status": "success", + "queue_types": queue_types, + "enqueue_counts": enqueue_counts, + "dequeue_counts": dequeue_counts, + } + ) + return response + elif queue_type and not queue_id: + pipe = self._r.pipeline() + pipe.zrange("%s:%s" % (self._key_prefix, queue_type), 0, -1) + pipe.zrange("%s:%s:active" % (self._key_prefix, queue_type), 0, -1) + ready_queues, active_queues = pipe.execute() + queue_list = format_queue_ids(ready_queues, active_queues) + response.update({"status": "success", "queue_ids": queue_list}) + return response + elif queue_type and queue_id: + timestamp = str(generate_epoch()) + keys = ["%s:%s:%s" % (self._key_prefix, queue_type, queue_id)] + args = [timestamp] + enqueue_details, dequeue_details = self._lua_metrics(keys=keys, args=args) + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, dequeue_details + ) + + queue_length = self._r.llen( + "%s:%s:%s" % (self._key_prefix, queue_type, queue_id) + ) + + response.update( + { + "status": "success", + "queue_length": int(queue_length), + "enqueue_counts": enqueue_counts, + "dequeue_counts": dequeue_counts, + } + ) + return response + elif not queue_type and queue_id: + raise BadArgumentException( + "`queue_id` should be accompanied by `queue_type`." + ) + + return response + + def deep_status(self): + """ + Check Redis availability. If Redis is down, set() will raise. + :return: value or None + """ + return self._r.set( + "fq:deep_status:{}".format(self._key_prefix), "sharq_deep_status" + ) + + def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): + """Clear entries in a queue and optionally purge related resources.""" + validate_clear_queue_arguments(queue_type, queue_id) + + response = {"status": "Failure", "message": "No queued calls found"} + primary_set = "{}:{}".format(self._key_prefix, queue_type) + queued_status = self._r.zrem(primary_set, queue_id) + if queued_status: + response.update( + { + "status": "Success", + "message": "Successfully removed all queued calls", + } + ) + + job_queue_list = "{}:{}:{}".format(self._key_prefix, queue_type, queue_id) + if queued_status and purge_all: + job_list = self._r.lrange(job_queue_list, 0, -1) + pipe = self._r.pipeline() + for job_uuid in job_list: + if job_uuid is None: + continue + job_uuid_str = decode_redis_value(job_uuid) + payload_set = "{}:payload".format(self._key_prefix) + job_payload_key = "{}:{}:{}".format(queue_type, queue_id, job_uuid_str) + pipe.hdel(payload_set, job_payload_key) + + interval_set = "{}:interval".format(self._key_prefix) + job_interval_key = "{}:{}".format(queue_type, queue_id) + pipe.hdel(interval_set, job_interval_key) + pipe.delete(job_queue_list) + pipe.execute() + response.update( + { + "status": "Success", + "message": "Successfully removed all queued calls and purged related resources", + } + ) + else: + self._r.delete(job_queue_list) + return response + + def get_queue_length(self, queue_type, queue_id): + """ + Return the current Redis list length for key_prefix:queue_type:queue_id. + """ + validate_get_queue_length_arguments(queue_type, queue_id) + + redis_key = self._key_prefix + ":" + queue_type + ":" + queue_id + return self._r.llen(redis_key) + + def close(self): + """Close the underlying synchronous Redis client.""" + if self._r is None: + return + + conn = self._r + close = getattr(conn, "close", None) + if callable(close): + close() + self._r = None + return + + pool = getattr(conn, "connection_pool", None) + if pool is not None: + disconnect = getattr(pool, "disconnect", None) + if callable(disconnect): + disconnect() + + self._r = None diff --git a/tests/test_sync_queue.py b/tests/test_sync_queue.py new file mode 100644 index 0000000..7616836 --- /dev/null +++ b/tests/test_sync_queue.py @@ -0,0 +1,352 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +import asyncio +import time +import unittest +import uuid + +from fq import FQ as AsyncFQ +from fq.exceptions import BadArgumentException +from fq.sync import FQ +from tests.config import build_test_config + + +class SyncFQTest(unittest.TestCase): + def setUp(self): + self.queue = FQ(build_test_config(redis={"key_prefix": "test_fq_sync"})) + self.queue.initialize() + self.queue._r.flushdb() + self.queue_type = "sms" + self.queue_id = "johndoe" + self.payload = {"to": "1000000000", "message": "Hello, sync FQ"} + + def tearDown(self): + if self.queue is not None and self.queue._r is not None: + self.queue._r.flushdb() + self.queue.close() + + def _job_id(self): + return str(uuid.uuid4()) + + def test_import_namespace(self): + from fq import FQ as ImportedAsyncFQ + from fq.sync import FQ as ImportedSyncFQ + + self.assertIs(ImportedAsyncFQ, AsyncFQ) + self.assertIs(ImportedSyncFQ, FQ) + self.assertIsNot(ImportedAsyncFQ, ImportedSyncFQ) + + def test_initialize_close_and_reload_scripts(self): + self.assertIs(self.queue.redis_client(), self.queue._r) + self.assertIsNotNone(self.queue._lua_enqueue) + + self.queue.reload_lua_scripts() + self.assertIsNotNone(self.queue._lua_enqueue) + self.assertTrue(self.queue.deep_status()) + + self.queue.close() + self.assertIsNone(self.queue._r) + self.queue.close() + + def test_enqueue_dequeue_finish(self): + job_id = self._job_id() + response = self.queue.enqueue( + payload=self.payload, + interval=1000, + job_id=job_id, + queue_id=self.queue_id, + queue_type=self.queue_type, + ) + self.assertEqual(response, {"status": "queued"}) + self.assertEqual(self.queue.get_queue_length(self.queue_type, self.queue_id), 1) + + job = self.queue.dequeue(queue_type=self.queue_type) + self.assertEqual( + job, + { + "status": "success", + "queue_id": self.queue_id, + "job_id": job_id, + "payload": self.payload, + "requeues_remaining": -1, + }, + ) + + self.assertEqual( + self.queue.finish( + queue_type=self.queue_type, + queue_id=job["queue_id"], + job_id=job["job_id"], + ), + {"status": "success"}, + ) + self.assertEqual( + self.queue.finish( + queue_type=self.queue_type, + queue_id=job["queue_id"], + job_id=job["job_id"], + ), + {"status": "failure"}, + ) + + def test_requeue_behavior(self): + self.queue.close() + self.queue = FQ( + build_test_config( + fq={"job_expire_interval": 20}, + redis={"key_prefix": "test_fq_sync_requeue"}, + ) + ) + self.queue.initialize() + self.queue._r.flushdb() + + job_id = self._job_id() + self.queue.enqueue( + payload=self.payload, + interval=1, + job_id=job_id, + queue_id=self.queue_id, + queue_type=self.queue_type, + requeue_limit=1, + ) + first_job = self.queue.dequeue(queue_type=self.queue_type) + self.assertEqual(first_job["status"], "success") + + time.sleep(0.08) + self.queue.requeue() + self.assertEqual(self.queue.get_queue_length(self.queue_type, self.queue_id), 1) + + requeued_job = self.queue.dequeue(queue_type=self.queue_type) + self.assertEqual(requeued_job["status"], "success") + self.assertEqual(requeued_job["job_id"], job_id) + self.assertEqual(requeued_job["requeues_remaining"], 0) + + def test_interval_update(self): + job_id = self._job_id() + self.queue.enqueue( + payload=self.payload, + interval=1000, + job_id=job_id, + queue_id=self.queue_id, + queue_type=self.queue_type, + ) + + response = self.queue.interval( + interval=250, + queue_id=self.queue_id, + queue_type=self.queue_type, + ) + self.assertEqual(response, {"status": "success"}) + self.assertEqual( + self.queue._r.hget( + "%s:interval" % self.queue._key_prefix, + "%s:%s" % (self.queue_type, self.queue_id), + ), + b"250", + ) + + def test_metrics(self): + response = self.queue.metrics() + self.assertEqual(response["status"], "success") + self.assertEqual(response["queue_types"], []) + self.assertEqual(sum(response["enqueue_counts"].values()), 0) + self.assertEqual(sum(response["dequeue_counts"].values()), 0) + + job_id = self._job_id() + self.queue.enqueue( + payload=self.payload, + interval=1, + job_id=job_id, + queue_id=self.queue_id, + queue_type=self.queue_type, + ) + + queue_type_metrics = self.queue.metrics(queue_type=self.queue_type) + self.assertEqual(queue_type_metrics["status"], "success") + self.assertEqual(queue_type_metrics["queue_ids"], [self.queue_id]) + + queue_metrics = self.queue.metrics( + queue_type=self.queue_type, + queue_id=self.queue_id, + ) + self.assertEqual(queue_metrics["status"], "success") + self.assertEqual(queue_metrics["queue_length"], 1) + self.assertEqual(sum(queue_metrics["enqueue_counts"].values()), 1) + + self.queue.dequeue(queue_type=self.queue_type) + global_metrics = self.queue.metrics() + self.assertEqual(global_metrics["queue_types"], [self.queue_type]) + self.assertEqual(sum(global_metrics["dequeue_counts"].values()), 1) + + def test_clear_queue(self): + job_id = self._job_id() + self.queue.enqueue( + payload=self.payload, + interval=1000, + job_id=job_id, + queue_id=self.queue_id, + queue_type=self.queue_type, + ) + + response = self.queue.clear_queue( + queue_type=self.queue_type, + queue_id=self.queue_id, + purge_all=True, + ) + self.assertEqual( + response, + { + "status": "Success", + "message": "Successfully removed all queued calls and purged related resources", + }, + ) + self.assertEqual(self.queue.get_queue_length(self.queue_type, self.queue_id), 0) + self.assertIsNone( + self.queue._r.hget( + "%s:interval" % self.queue._key_prefix, + "%s:%s" % (self.queue_type, self.queue_id), + ) + ) + + def test_validation_errors_match_async_api(self): + def collect_sync_errors(): + checks = [ + lambda: self.queue.enqueue( + payload=self.payload, + interval=0, + job_id=self._job_id(), + queue_id=self.queue_id, + queue_type=self.queue_type, + ), + lambda: self.queue.dequeue(queue_type="bad type"), + lambda: self.queue.finish( + job_id="bad id", + queue_id=self.queue_id, + queue_type=self.queue_type, + ), + lambda: self.queue.interval( + interval=0, + queue_id=self.queue_id, + queue_type=self.queue_type, + ), + lambda: self.queue.metrics(queue_id=self.queue_id), + lambda: self.queue.clear_queue( + queue_type=self.queue_type, + queue_id="bad id", + ), + lambda: self.queue.get_queue_length("bad type", self.queue_id), + ] + errors = [] + for check in checks: + with self.assertRaises(BadArgumentException) as ctx: + check() + errors.append(str(ctx.exception)) + return errors + + async def collect_async_errors(): + queue = AsyncFQ(build_test_config(redis={"key_prefix": "test_fq_async"})) + await queue.initialize() + await queue._r.flushdb() + checks = [ + lambda: queue.enqueue( + payload=self.payload, + interval=0, + job_id=self._job_id(), + queue_id=self.queue_id, + queue_type=self.queue_type, + ), + lambda: queue.dequeue(queue_type="bad type"), + lambda: queue.finish( + job_id="bad id", + queue_id=self.queue_id, + queue_type=self.queue_type, + ), + lambda: queue.interval( + interval=0, + queue_id=self.queue_id, + queue_type=self.queue_type, + ), + lambda: queue.metrics(queue_id=self.queue_id), + lambda: queue.clear_queue( + queue_type=self.queue_type, + queue_id="bad id", + ), + lambda: queue.get_queue_length("bad type", self.queue_id), + ] + errors = [] + try: + for check in checks: + with self.assertRaises(BadArgumentException) as ctx: + await check() + errors.append(str(ctx.exception)) + return errors + finally: + await queue._r.flushdb() + await queue.close() + + self.assertEqual(collect_sync_errors(), asyncio.run(collect_async_errors())) + + def test_sync_async_interoperability(self): + async def scenario(): + config = build_test_config(redis={"key_prefix": "test_fq_sync_interop"}) + async_queue = AsyncFQ(config) + sync_queue = FQ(config) + + await async_queue.initialize() + sync_queue.initialize() + await async_queue._r.flushdb() + + try: + sync_job_id = self._job_id() + sync_queue.enqueue( + payload={"source": "sync"}, + interval=1, + job_id=sync_job_id, + queue_id=self.queue_id, + queue_type=self.queue_type, + ) + sync_job = await async_queue.dequeue(queue_type=self.queue_type) + self.assertEqual(sync_job["status"], "success") + self.assertEqual(sync_job["job_id"], sync_job_id) + self.assertEqual(sync_job["payload"], {"source": "sync"}) + self.assertEqual( + await async_queue.finish( + queue_type=self.queue_type, + queue_id=sync_job["queue_id"], + job_id=sync_job["job_id"], + ), + {"status": "success"}, + ) + + async_job_id = self._job_id() + await async_queue.enqueue( + payload={"source": "async"}, + interval=1, + job_id=async_job_id, + queue_id=self.queue_id, + queue_type=self.queue_type, + ) + await asyncio.sleep(0.01) + async_job = sync_queue.dequeue(queue_type=self.queue_type) + self.assertEqual(async_job["status"], "success") + self.assertEqual(async_job["job_id"], async_job_id) + self.assertEqual(async_job["payload"], {"source": "async"}) + self.assertEqual( + sync_queue.finish( + queue_type=self.queue_type, + queue_id=async_job["queue_id"], + job_id=async_job["job_id"], + ), + {"status": "success"}, + ) + finally: + sync_queue._r.flushdb() + sync_queue.close() + await async_queue.close() + + asyncio.run(scenario()) + + +if __name__ == "__main__": + unittest.main() From d631fc03418c782d3186cf71251fb7069bd1ea73 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 13:04:16 +0100 Subject: [PATCH 02/12] Adds code quality tool config and minor code cleanup Introduces configuration files for a code quality toolkit, enabling code health checks and exclusions. Simplifies exception handling by removing unused variables and cleans up redundant test await usage to improve code clarity. --- .qlty/.gitignore | 7 ++++ .qlty/qlty.toml | 88 ++++++++++++++++++++++++++++++++++++++++++++++ src/fq/utils.py | 2 +- tests/test_func.py | 2 +- 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 .qlty/.gitignore create mode 100644 .qlty/qlty.toml diff --git a/.qlty/.gitignore b/.qlty/.gitignore new file mode 100644 index 0000000..3036618 --- /dev/null +++ b/.qlty/.gitignore @@ -0,0 +1,7 @@ +* +!configs +!configs/** +!hooks +!hooks/** +!qlty.toml +!.gitignore diff --git a/.qlty/qlty.toml b/.qlty/qlty.toml new file mode 100644 index 0000000..f5140b3 --- /dev/null +++ b/.qlty/qlty.toml @@ -0,0 +1,88 @@ +# This file was automatically generated by `qlty init`. +# You can modify it to suit your needs. +# We recommend you to commit this file to your repository. +# +# This configuration is used by both Qlty CLI and Qlty Cloud. +# +# Qlty CLI -- Code quality toolkit for developers +# Qlty Cloud -- Fully automated Code Health Platform +# +# Try Qlty Cloud: https://qlty.sh +# +# For a guide to configuration, visit https://qlty.sh/d/config +# Or for a full reference, visit https://qlty.sh/d/qlty-toml +config_version = "0" + +exclude_patterns = [ + "*_min.*", + "*-min.*", + "*.min.*", + "**/.yarn/**", + "**/*.d.ts", + "**/assets/**", + "**/bower_components/**", + "**/build/**", + "**/cache/**", + "**/config/**", + "**/db/**", + "**/deps/**", + "**/dist/**", + "**/extern/**", + "**/external/**", + "**/generated/**", + "**/Godeps/**", + "**/gradlew/**", + "**/mvnw/**", + "**/node_modules/**", + "**/protos/**", + "**/seed/**", + "**/target/**", + "**/templates/**", + "**/testdata/**", + "**/vendor/**", +] + +test_patterns = [ + "**/test/**", + "**/spec/**", + "**/*.test.*", + "**/*.spec.*", + "**/*_test.*", + "**/*_spec.*", + "**/test_*.*", + "**/spec_*.*", +] + +[smells] +mode = "comment" + +[[source]] +name = "default" +default = true + + +[[plugin]] +name = "actionlint" + +[[plugin]] +name = "bandit" + +[[plugin]] +name = "radarlint-python" +mode = "comment" + +[[plugin]] +name = "ripgrep" +mode = "comment" + +[[plugin]] +name = "ruff" +drivers = [ + "lint", +] + +[[plugin]] +name = "trufflehog" + +[[plugin]] +name = "zizmor" diff --git a/src/fq/utils.py b/src/fq/utils.py index 66fb642..ba7d719 100644 --- a/src/fq/utils.py +++ b/src/fq/utils.py @@ -76,7 +76,7 @@ def convert_to_str(queue_set): for queue in list(queue_set): try: queue_list.append(queue.decode("utf-8")) - except Exception as e: + except Exception: queue_list.append(queue) pass return queue_list diff --git a/tests/test_func.py b/tests/test_func.py index a2035d6..9580809 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -712,7 +712,7 @@ async def test_dequeue_time_keeper_existence(self): queue_id=self._test_queue_id, queue_type=self._test_queue_type, ) - response = await self.queue.dequeue(queue_type=self._test_queue_type) + await self.queue.dequeue(queue_type=self._test_queue_type) time_keeper_key_name = "%s:%s:%s:time" % ( self.queue._key_prefix, From 926228c1dc9fd3375dd827293799ad236e9acd11 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 13:05:49 +0100 Subject: [PATCH 03/12] Removes unused time import and redundant variable assignments in test cases --- tests/test_func.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_func.py b/tests/test_func.py index 9580809..bbc491a 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Copyright (c) 2014 Plivo Team. See LICENSE.txt for details. import uuid -import time import math import asyncio import unittest @@ -606,7 +605,7 @@ async def test_enqueue_second_job_queue_type_ready_set(self): ) # job 2 job_id = self._get_job_id() - start_time = str(generate_epoch()) + str(generate_epoch()) await self.queue.enqueue( payload=self._test_payload_2, interval=20000, @@ -634,7 +633,7 @@ async def test_enqueue_second_job_queue_type_active_set(self): ) # job 2 job_id = self._get_job_id() - start_time = str(generate_epoch()) + str(generate_epoch()) await self.queue.enqueue( payload=self._test_payload_2, interval=20000, From 2c19e098570c88d3f4b4f133ef898aac6c48d83a Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 13:30:01 +0100 Subject: [PATCH 04/12] Pins GitHub Actions to specific commit SHAs Enhances CI security and reproducibility by replacing version tags with explicit commit SHAs for all third-party actions in workflow files. Also disables credential persistence for checkouts to reduce potential risk exposure. --- .github/workflows/pypi.yml | 7 ++++--- .github/workflows/tests.yml | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index c0b3d47..b5405af 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -18,12 +18,13 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: fetch-depth: 0 + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.12" @@ -33,4 +34,4 @@ jobs: python -m build - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 48ce0b4..704f1c6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -27,10 +27,12 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - name: Install uv (with Python 3.12) - uses: astral-sh/setup-uv@v4 + uses: astral-sh/setup-uv@e4db8464a088ece1b920f60402e813ea4de65b8f # v4 with: python-version: "3.12" @@ -45,7 +47,7 @@ jobs: uv run pytest --cov=src --cov-branch --cov-report=xml - name: Upload results to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5 with: token: ${{ secrets.CODECOV_TOKEN }} slug: flowdacity/queue-engine From 283e95565265ed9fd5f1fae48da59c77ed43d486 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 13:30:15 +0100 Subject: [PATCH 05/12] Refactors config validation and streamlines tests Modularizes and clarifies configuration validation logic, introducing helper functions and improving error messaging for invalid arguments. Enhances test reliability by using platform-agnostic temp directories for Unix socket paths and suppressing teardown exceptions. Annotates unittest lifecycle hooks to quiet static analysis warnings. --- src/fq/core.py | 198 ++++++++++++++++++++++----------------- src/fq/utils.py | 1 - tests/config.py | 7 +- tests/test_edge_cases.py | 14 +-- tests/test_func.py | 12 ++- tests/test_queue.py | 2 + 6 files changed, 133 insertions(+), 101 deletions(-) diff --git a/src/fq/core.py b/src/fq/core.py index afaaeb0..4e4132d 100644 --- a/src/fq/core.py +++ b/src/fq/core.py @@ -16,9 +16,31 @@ ) LUA_SCRIPT_NAMES = ("enqueue", "dequeue", "finish", "interval", "requeue", "metrics") +REDIS_CONN_TYPES = {"tcp_sock", "unix_sock"} + +INVALID_INTERVAL = "`interval` has an invalid value." +INVALID_JOB_ID = "`job_id` has an invalid value." +INVALID_QUEUE_ID = "`queue_id` has an invalid value." +INVALID_QUEUE_TYPE = "`queue_type` has an invalid value." +INVALID_REQUEUE_LIMIT = "`requeue_limit` has an invalid value." def normalize_config(config): + normalized = _normalize_config_sections(config) + _require_config_sections(normalized) + + redis_config = normalized["redis"] + fq_config = normalized["fq"] + + _validate_redis_config(redis_config) + _validate_fq_config(fq_config) + _validate_connection_config(redis_config) + _validate_optional_redis_config(redis_config) + + return normalized + + +def _normalize_config_sections(config): if not isinstance(config, Mapping): raise FQException("Config must be a mapping with redis and fq sections") @@ -31,85 +53,102 @@ def normalize_config(config): str(option): value for option, value in section_values.items() } - if "redis" not in normalized or "fq" not in normalized: + return normalized + + +def _require_config_sections(config): + if "redis" not in config or "fq" not in config: raise FQException("Config missing required sections: redis, fq") - redis_config = normalized["redis"] - fq_config = normalized["fq"] - if "key_prefix" not in redis_config: - raise FQException("Missing config: redis.key_prefix") - if not isinstance(redis_config["key_prefix"], str) or not redis_config[ - "key_prefix" - ]: +def _require_config_value(config, section_name, option_name): + if option_name not in config: + raise FQException("Missing config: %s.%s" % (section_name, option_name)) + + return config[option_name] + + +def _is_non_empty_string(value): + return isinstance(value, str) and bool(value) + + +def _is_int_not_bool(value): + return isinstance(value, int) and not isinstance(value, bool) + + +def _validate_redis_config(redis_config): + key_prefix = _require_config_value(redis_config, "redis", "key_prefix") + if not _is_non_empty_string(key_prefix): raise FQException("Invalid config: redis.key_prefix must be a non-empty string") - if "conn_type" not in redis_config: - raise FQException("Missing config: redis.conn_type") - if redis_config["conn_type"] not in {"tcp_sock", "unix_sock"}: + conn_type = _require_config_value(redis_config, "redis", "conn_type") + if conn_type not in REDIS_CONN_TYPES: raise FQException( "Invalid config: redis.conn_type must be 'tcp_sock' or 'unix_sock'" ) - if "db" not in redis_config: - raise FQException("Missing config: redis.db") - if isinstance(redis_config["db"], bool) or not isinstance(redis_config["db"], int): + db = _require_config_value(redis_config, "redis", "db") + if not _is_int_not_bool(db): raise FQException("Invalid config: redis.db must be an integer") - if "job_expire_interval" not in fq_config: - raise FQException("Missing config: fq.job_expire_interval") - if not is_valid_interval(fq_config["job_expire_interval"]): - raise FQException( - "Invalid config: fq.job_expire_interval must be a positive integer" - ) - if "job_requeue_interval" not in fq_config: - raise FQException("Missing config: fq.job_requeue_interval") - if not is_valid_interval(fq_config["job_requeue_interval"]): - raise FQException( - "Invalid config: fq.job_requeue_interval must be a positive integer" - ) +def _validate_fq_config(fq_config): + for option_name in ("job_expire_interval", "job_requeue_interval"): + value = _require_config_value(fq_config, "fq", option_name) + if not is_valid_interval(value): + raise FQException( + "Invalid config: fq.%s must be a positive integer" % option_name + ) - if "default_job_requeue_limit" not in fq_config: - raise FQException("Missing config: fq.default_job_requeue_limit") - if not is_valid_requeue_limit(fq_config["default_job_requeue_limit"]): + default_requeue_limit = _require_config_value( + fq_config, "fq", "default_job_requeue_limit" + ) + if not is_valid_requeue_limit(default_requeue_limit): raise FQException( "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" ) + +def _validate_connection_config(redis_config): if redis_config["conn_type"] == "unix_sock": - if "unix_socket_path" not in redis_config: - raise FQException("Missing config: redis.unix_socket_path") - if not isinstance(redis_config["unix_socket_path"], str) or not redis_config[ - "unix_socket_path" - ]: - raise FQException( - "Invalid config: redis.unix_socket_path must be a non-empty string" - ) + _validate_unix_socket_config(redis_config) + return + + _validate_tcp_socket_config(redis_config) + - if redis_config["conn_type"] == "tcp_sock": - if "host" not in redis_config: - raise FQException("Missing config: redis.host") - if not isinstance(redis_config["host"], str) or not redis_config["host"]: - raise FQException("Invalid config: redis.host must be a non-empty string") +def _validate_unix_socket_config(redis_config): + unix_socket_path = _require_config_value( + redis_config, "redis", "unix_socket_path" + ) + if not _is_non_empty_string(unix_socket_path): + raise FQException( + "Invalid config: redis.unix_socket_path must be a non-empty string" + ) - if "port" not in redis_config: - raise FQException("Missing config: redis.port") - if isinstance(redis_config["port"], bool) or not isinstance( - redis_config["port"], int - ): - raise FQException("Invalid config: redis.port must be an integer") - if "clustered" in redis_config and not isinstance( - redis_config["clustered"], bool - ): - raise FQException("Invalid config: redis.clustered must be a boolean") +def _validate_tcp_socket_config(redis_config): + host = _require_config_value(redis_config, "redis", "host") + if not _is_non_empty_string(host): + raise FQException("Invalid config: redis.host must be a non-empty string") + port = _require_config_value(redis_config, "redis", "port") + if not _is_int_not_bool(port): + raise FQException("Invalid config: redis.port must be an integer") + + if "clustered" in redis_config and not isinstance(redis_config["clustered"], bool): + raise FQException("Invalid config: redis.clustered must be a boolean") + + +def _validate_optional_redis_config(redis_config): if "password" in redis_config and redis_config["password"] is not None: if not isinstance(redis_config["password"], str): raise FQException("Invalid config: redis.password must be a string") - return normalized + +def _validate_identifier(identifier, message): + if not is_valid_identifier(identifier): + raise BadArgumentException(message) def load_lua_scripts(instance, redis_client): @@ -142,22 +181,17 @@ def validate_enqueue_arguments( default_requeue_limit, ): if not is_valid_interval(interval): - raise BadArgumentException("`interval` has an invalid value.") - - if not is_valid_identifier(job_id): - raise BadArgumentException("`job_id` has an invalid value.") + raise BadArgumentException(INVALID_INTERVAL) - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + _validate_identifier(job_id, INVALID_JOB_ID) + _validate_identifier(queue_id, INVALID_QUEUE_ID) + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) if requeue_limit is None: requeue_limit = default_requeue_limit if not is_valid_requeue_limit(requeue_limit): - raise BadArgumentException("`requeue_limit` has an invalid value.") + raise BadArgumentException(INVALID_REQUEUE_LIMIT) try: serialized_payload = serialize_payload(payload) @@ -168,54 +202,42 @@ def validate_enqueue_arguments( def validate_dequeue_arguments(queue_type): - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) def validate_finish_arguments(job_id, queue_id, queue_type): - if not is_valid_identifier(job_id): - raise BadArgumentException("`job_id` has an invalid value.") - - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + _validate_identifier(job_id, INVALID_JOB_ID) + _validate_identifier(queue_id, INVALID_QUEUE_ID) + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) def validate_interval_arguments(interval, queue_id, queue_type): if not is_valid_interval(interval): - raise BadArgumentException("`interval` has an invalid value.") + raise BadArgumentException(INVALID_INTERVAL) - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") - - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + _validate_identifier(queue_id, INVALID_QUEUE_ID) + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) def validate_metrics_arguments(queue_type, queue_id): if queue_id is not None and not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") + raise BadArgumentException(INVALID_QUEUE_ID) if queue_type is not None and not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + raise BadArgumentException(INVALID_QUEUE_TYPE) def validate_clear_queue_arguments(queue_type, queue_id): if queue_id is None or not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") + raise BadArgumentException(INVALID_QUEUE_ID) if queue_type is None or not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") + raise BadArgumentException(INVALID_QUEUE_TYPE) def validate_get_queue_length_arguments(queue_type, queue_id): - if not is_valid_identifier(queue_type): - raise BadArgumentException("`queue_type` has an invalid value.") - - if not is_valid_identifier(queue_id): - raise BadArgumentException("`queue_id` has an invalid value.") + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) + _validate_identifier(queue_id, INVALID_QUEUE_ID) def decode_redis_value(value): diff --git a/src/fq/utils.py b/src/fq/utils.py index ba7d719..11faad3 100644 --- a/src/fq/utils.py +++ b/src/fq/utils.py @@ -78,5 +78,4 @@ def convert_to_str(queue_set): queue_list.append(queue.decode("utf-8")) except Exception: queue_list.append(queue) - pass return queue_list diff --git a/tests/config.py b/tests/config.py index 617030c..83e1e6b 100644 --- a/tests/config.py +++ b/tests/config.py @@ -1,6 +1,11 @@ # -*- coding: utf-8 -*- from copy import deepcopy +from os.path import join +from tempfile import gettempdir + + +TEST_UNIX_SOCKET_PATH = join(gettempdir(), "redis.sock") TEST_CONFIG = { @@ -13,7 +18,7 @@ "db": 0, "key_prefix": "test_fq", "conn_type": "tcp_sock", - "unix_socket_path": "/tmp/redis.sock", + "unix_socket_path": TEST_UNIX_SOCKET_PATH, "port": 6379, "host": "127.0.0.1", "clustered": False, diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index b84a51d..eef44f4 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -3,6 +3,7 @@ import unittest +from contextlib import suppress from unittest.mock import patch from fq import FQ @@ -55,7 +56,8 @@ async def set(self, key, value): class FakeRedisConnectionFailure: def __init__(self, *args, **kwargs): - pass + self.init_args = args + self.init_kwargs = kwargs async def ping(self): raise ConnectionError("boom") @@ -111,27 +113,25 @@ async def delete(self, key): class TestEdgeCases(unittest.IsolatedAsyncioTestCase): + # qlty-ignore(radarlint-python:python:S5899): unittest lifecycle hook. async def asyncSetUp(self): self.config = build_test_config() self.fq_instance = None + # qlty-ignore(radarlint-python:python:S5899): unittest lifecycle hook. async def asyncTearDown(self): """Clean up Redis state and close connections after each test.""" # If a test initialized FQ with real Redis, clean up if self.fq_instance is not None: - try: + with suppress(Exception): if self.fq_instance._r is not None: await self.fq_instance._r.flushdb() await self.fq_instance.close() - except Exception: - # Ignore errors during cleanup - tests may have mocked or closed connections - # This prevents tearDown failures from masking test failures - pass self.fq_instance = None def test_invalid_config_type_raises(self): with self.assertRaisesRegex(FQException, "Config must be a mapping"): - FQ("/tmp/does-not-exist.conf") + FQ("does-not-exist.conf") async def test_initialize_fails_fast_on_bad_redis(self): with patch("fq.queue.Redis", FakeRedisConnectionFailure): diff --git a/tests/test_func.py b/tests/test_func.py index bbc491a..d964fd7 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -5,6 +5,8 @@ import asyncio import unittest import msgpack +from os.path import join +from tempfile import gettempdir from unittest.mock import AsyncMock, MagicMock from fq import FQ from fq.exceptions import FQException @@ -12,6 +14,8 @@ from tests.config import build_test_config +NONEXISTENT_UNIX_SOCKET_PATH = join(gettempdir(), "redis_nonexistent.sock") + class FQTestCase(unittest.IsolatedAsyncioTestCase): """ @@ -20,6 +24,7 @@ class FQTestCase(unittest.IsolatedAsyncioTestCase): by FQ. """ + # qlty-ignore(radarlint-python:python:S5899): unittest lifecycle hook. async def asyncSetUp(self): self.queue = FQ(build_test_config()) # flush all the keys in the test db before starting test @@ -605,7 +610,6 @@ async def test_enqueue_second_job_queue_type_ready_set(self): ) # job 2 job_id = self._get_job_id() - str(generate_epoch()) await self.queue.enqueue( payload=self._test_payload_2, interval=20000, @@ -633,7 +637,6 @@ async def test_enqueue_second_job_queue_type_active_set(self): ) # job 2 job_id = self._get_job_id() - str(generate_epoch()) await self.queue.enqueue( payload=self._test_payload_2, interval=20000, @@ -1811,7 +1814,7 @@ async def test_initialize_unix_socket_connection(self): redis={ "key_prefix": "test_fq_unix", "conn_type": "unix_sock", - "unix_socket_path": "/tmp/redis_nonexistent.sock", + "unix_socket_path": NONEXISTENT_UNIX_SOCKET_PATH, } ) @@ -1835,7 +1838,7 @@ def mock_redis_constructor(**kwargs): # Verify that Redis was initialized with unix_socket_path self.assertIn("unix_socket_path", redis_init_kwargs) self.assertEqual( - redis_init_kwargs["unix_socket_path"], "/tmp/redis_nonexistent.sock" + redis_init_kwargs["unix_socket_path"], NONEXISTENT_UNIX_SOCKET_PATH ) self.assertEqual(int(redis_init_kwargs["db"]), 0) @@ -1915,6 +1918,7 @@ async def test_convert_to_str_with_mixed_types(self): self.assertIn("key2", result) self.assertIn("key3", result) + # qlty-ignore(radarlint-python:python:S5899): unittest lifecycle hook. async def asyncTearDown(self): await self.queue._r.flushdb() await self.queue.close() diff --git a/tests/test_queue.py b/tests/test_queue.py index e94a2f0..1f5c1d6 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -11,6 +11,7 @@ class FQTest(unittest.IsolatedAsyncioTestCase): """The FQTest contains test cases which validate the FQ interface.""" + # qlty-ignore(radarlint-python:python:S5899): unittest lifecycle hook. async def asyncSetUp(self): self.queue = FQ(build_test_config()) await self.queue.initialize() @@ -58,6 +59,7 @@ async def asyncSetUp(self): # flush redis before start await self.queue._r.flushdb() + # qlty-ignore(radarlint-python:python:S5899): unittest lifecycle hook. async def asyncTearDown(self): # flush redis at the end and close connection await self.queue._r.flushdb() From d5b19f0d81d9e0ea3388a98f334381de928901a1 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 19:38:36 +0100 Subject: [PATCH 06/12] Refactors core logic and modularizes queue internals Splits monolithic core into focused modules for config, Redis operations, Lua script loading, response formatting, and argument validation. Improves code maintainability, testability, and clarity by decoupling responsibilities. Updates async and sync queue classes to use the refactored internals and unifies key construction across code paths. Adjusts tests for new object structure and patch targets. Enables easier extension and more reliable Redis and Lua handling by isolating connection logic and script registration. Moves config validation and argument checking to dedicated modules, reducing duplication. No functional behavior is changed, but future enhancements and bug fixes are now simpler and safer. --- README.md | 16 +- src/fq/config.py | 164 +++++++++++++++++++ src/fq/core.py | 311 ----------------------------------- src/fq/keys.py | 44 +++++ src/fq/lua.py | 33 ++++ src/fq/queue.py | 345 ++++++++++++--------------------------- src/fq/redis.py | 100 ++++++++++++ src/fq/responses.py | 51 ++++++ src/fq/sync/queue.py | 216 ++++++++++-------------- src/fq/validators.py | 102 ++++++++++++ tests/test_edge_cases.py | 14 +- tests/test_func.py | 7 +- tests/test_sync_queue.py | 24 ++- 13 files changed, 730 insertions(+), 697 deletions(-) create mode 100644 src/fq/config.py delete mode 100644 src/fq/core.py create mode 100644 src/fq/keys.py create mode 100644 src/fq/lua.py create mode 100644 src/fq/redis.py create mode 100644 src/fq/responses.py create mode 100644 src/fq/validators.py diff --git a/README.md b/README.md index f8675c1..f8a4855 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ Flowdacity Queue ================ -Flowdacity Queue (FQ) is a rate-limited job queue built on Redis with supported async and sync APIs. It stores jobs per queue type and queue id, enforces per-queue dequeue intervals, automatically requeues expired jobs, and exposes metrics to understand throughput and queue depth. +Flowdacity Queue (FQ) is a rate-limited job queue built on Redis. It stores jobs per queue type and queue id, enforces per-queue dequeue intervals, automatically requeues expired jobs, and exposes metrics to understand throughput and queue depth. ## Features - Per-queue rate limiting using millisecond intervals. -- Async and sync APIs backed by Redis clients from redis-py. -- Lua scripts for predictable queue behavior across both APIs. +- Async and sync interfaces backed by Redis clients from redis-py. +- Lua scripts for predictable queue behavior. - Automatic retries with configurable limits (including infinite retries). - Metrics for enqueue/dequeue counts and queue lengths. - Works with TCP or Unix socket Redis deployments and supports Redis Cluster. @@ -61,9 +61,9 @@ config = { > unixsocketperm 755 > ``` -## Async API +## Async Usage -The top-level namespace is async-first: +Import `FQ` from the top-level package: ```python from fq import FQ @@ -121,9 +121,9 @@ async def main(): asyncio.run(main()) ``` -## Sync API +## Sync Usage -The sync API is available from the `fq.sync` namespace and supports the same queue operations without `await`: +Import `FQ` from `fq.sync`: ```python import uuid @@ -178,7 +178,7 @@ fq.close() - `await fq.metrics()` — global metrics; pass `queue_type` and/or `queue_id` for scoped stats and queue length. - `await fq.clear_queue(queue_type="sms", queue_id="user001", purge_all=True)` — drop queued jobs and their payload/interval metadata. -For sync code, use the same method names without `await`: `fq.requeue()`, `fq.interval(...)`, `fq.metrics()`, and `fq.clear_queue(...)`. +The same operations are available from `fq.sync.FQ` without `await`. ## Development diff --git a/src/fq/config.py b/src/fq/config.py new file mode 100644 index 0000000..6a96dd4 --- /dev/null +++ b/src/fq/config.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from collections.abc import Mapping +from dataclasses import dataclass + +from fq.exceptions import FQException +from fq.utils import is_valid_interval, is_valid_requeue_limit + + +REDIS_CONN_TYPES = {"tcp_sock", "unix_sock"} + + +@dataclass(frozen=True) +class RedisConfig: + key_prefix: str + conn_type: str + db: int + host: str | None = None + port: int | None = None + unix_socket_path: str | None = None + clustered: bool = False + password: str | None = None + + +@dataclass(frozen=True) +class FQConfig: + redis: RedisConfig + job_expire_interval: int + job_requeue_interval: int + default_job_requeue_limit: int + + @classmethod + def from_mapping(cls, config): + normalized = _normalize_config_sections(config) + _require_config_sections(normalized) + + redis_config = normalized["redis"] + fq_config = normalized["fq"] + + _validate_redis_config(redis_config) + _validate_fq_config(fq_config) + _validate_connection_config(redis_config) + _validate_optional_redis_config(redis_config) + + return cls( + redis=RedisConfig( + key_prefix=redis_config["key_prefix"], + conn_type=redis_config["conn_type"], + db=redis_config["db"], + host=redis_config.get("host"), + port=redis_config.get("port"), + unix_socket_path=redis_config.get("unix_socket_path"), + clustered=redis_config.get("clustered", False), + password=redis_config.get("password"), + ), + job_expire_interval=fq_config["job_expire_interval"], + job_requeue_interval=fq_config["job_requeue_interval"], + default_job_requeue_limit=fq_config["default_job_requeue_limit"], + ) + + +def _normalize_config_sections(config): + if not isinstance(config, Mapping): + raise FQException("Config must be a mapping with redis and fq sections") + + normalized = {} + for section_name, section_values in config.items(): + if not isinstance(section_values, Mapping): + raise FQException("Config section '%s' must be a mapping" % section_name) + + normalized[str(section_name)] = { + str(option): value for option, value in section_values.items() + } + + return normalized + + +def _require_config_sections(config): + if "redis" not in config or "fq" not in config: + raise FQException("Config missing required sections: redis, fq") + + +def _require_config_value(config, section_name, option_name): + if option_name not in config: + raise FQException("Missing config: %s.%s" % (section_name, option_name)) + + return config[option_name] + + +def _is_non_empty_string(value): + return isinstance(value, str) and bool(value) + + +def _is_int_not_bool(value): + return isinstance(value, int) and not isinstance(value, bool) + + +def _validate_redis_config(redis_config): + key_prefix = _require_config_value(redis_config, "redis", "key_prefix") + if not _is_non_empty_string(key_prefix): + raise FQException("Invalid config: redis.key_prefix must be a non-empty string") + + conn_type = _require_config_value(redis_config, "redis", "conn_type") + if conn_type not in REDIS_CONN_TYPES: + raise FQException( + "Invalid config: redis.conn_type must be 'tcp_sock' or 'unix_sock'" + ) + + db = _require_config_value(redis_config, "redis", "db") + if not _is_int_not_bool(db): + raise FQException("Invalid config: redis.db must be an integer") + + +def _validate_fq_config(fq_config): + for option_name in ("job_expire_interval", "job_requeue_interval"): + value = _require_config_value(fq_config, "fq", option_name) + if not is_valid_interval(value): + raise FQException( + "Invalid config: fq.%s must be a positive integer" % option_name + ) + + default_requeue_limit = _require_config_value( + fq_config, "fq", "default_job_requeue_limit" + ) + if not is_valid_requeue_limit(default_requeue_limit): + raise FQException( + "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" + ) + + +def _validate_connection_config(redis_config): + if redis_config["conn_type"] == "unix_sock": + _validate_unix_socket_config(redis_config) + return + + _validate_tcp_socket_config(redis_config) + + +def _validate_unix_socket_config(redis_config): + unix_socket_path = _require_config_value(redis_config, "redis", "unix_socket_path") + if not _is_non_empty_string(unix_socket_path): + raise FQException( + "Invalid config: redis.unix_socket_path must be a non-empty string" + ) + + +def _validate_tcp_socket_config(redis_config): + host = _require_config_value(redis_config, "redis", "host") + if not _is_non_empty_string(host): + raise FQException("Invalid config: redis.host must be a non-empty string") + + port = _require_config_value(redis_config, "redis", "port") + if not _is_int_not_bool(port): + raise FQException("Invalid config: redis.port must be an integer") + + if "clustered" in redis_config and not isinstance(redis_config["clustered"], bool): + raise FQException("Invalid config: redis.clustered must be a boolean") + + +def _validate_optional_redis_config(redis_config): + if "password" in redis_config and redis_config["password"] is not None: + if not isinstance(redis_config["password"], str): + raise FQException("Invalid config: redis.password must be a string") diff --git a/src/fq/core.py b/src/fq/core.py deleted file mode 100644 index 4e4132d..0000000 --- a/src/fq/core.py +++ /dev/null @@ -1,311 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. - -import os -from collections.abc import Mapping - -from fq.exceptions import BadArgumentException, FQException -from fq.utils import ( - convert_to_str, - deserialize_payload, - generate_epoch, - is_valid_identifier, - is_valid_interval, - is_valid_requeue_limit, - serialize_payload, -) - -LUA_SCRIPT_NAMES = ("enqueue", "dequeue", "finish", "interval", "requeue", "metrics") -REDIS_CONN_TYPES = {"tcp_sock", "unix_sock"} - -INVALID_INTERVAL = "`interval` has an invalid value." -INVALID_JOB_ID = "`job_id` has an invalid value." -INVALID_QUEUE_ID = "`queue_id` has an invalid value." -INVALID_QUEUE_TYPE = "`queue_type` has an invalid value." -INVALID_REQUEUE_LIMIT = "`requeue_limit` has an invalid value." - - -def normalize_config(config): - normalized = _normalize_config_sections(config) - _require_config_sections(normalized) - - redis_config = normalized["redis"] - fq_config = normalized["fq"] - - _validate_redis_config(redis_config) - _validate_fq_config(fq_config) - _validate_connection_config(redis_config) - _validate_optional_redis_config(redis_config) - - return normalized - - -def _normalize_config_sections(config): - if not isinstance(config, Mapping): - raise FQException("Config must be a mapping with redis and fq sections") - - normalized = {} - for section_name, section_values in config.items(): - if not isinstance(section_values, Mapping): - raise FQException("Config section '%s' must be a mapping" % section_name) - - normalized[str(section_name)] = { - str(option): value for option, value in section_values.items() - } - - return normalized - - -def _require_config_sections(config): - if "redis" not in config or "fq" not in config: - raise FQException("Config missing required sections: redis, fq") - - -def _require_config_value(config, section_name, option_name): - if option_name not in config: - raise FQException("Missing config: %s.%s" % (section_name, option_name)) - - return config[option_name] - - -def _is_non_empty_string(value): - return isinstance(value, str) and bool(value) - - -def _is_int_not_bool(value): - return isinstance(value, int) and not isinstance(value, bool) - - -def _validate_redis_config(redis_config): - key_prefix = _require_config_value(redis_config, "redis", "key_prefix") - if not _is_non_empty_string(key_prefix): - raise FQException("Invalid config: redis.key_prefix must be a non-empty string") - - conn_type = _require_config_value(redis_config, "redis", "conn_type") - if conn_type not in REDIS_CONN_TYPES: - raise FQException( - "Invalid config: redis.conn_type must be 'tcp_sock' or 'unix_sock'" - ) - - db = _require_config_value(redis_config, "redis", "db") - if not _is_int_not_bool(db): - raise FQException("Invalid config: redis.db must be an integer") - - -def _validate_fq_config(fq_config): - for option_name in ("job_expire_interval", "job_requeue_interval"): - value = _require_config_value(fq_config, "fq", option_name) - if not is_valid_interval(value): - raise FQException( - "Invalid config: fq.%s must be a positive integer" % option_name - ) - - default_requeue_limit = _require_config_value( - fq_config, "fq", "default_job_requeue_limit" - ) - if not is_valid_requeue_limit(default_requeue_limit): - raise FQException( - "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" - ) - - -def _validate_connection_config(redis_config): - if redis_config["conn_type"] == "unix_sock": - _validate_unix_socket_config(redis_config) - return - - _validate_tcp_socket_config(redis_config) - - -def _validate_unix_socket_config(redis_config): - unix_socket_path = _require_config_value( - redis_config, "redis", "unix_socket_path" - ) - if not _is_non_empty_string(unix_socket_path): - raise FQException( - "Invalid config: redis.unix_socket_path must be a non-empty string" - ) - - -def _validate_tcp_socket_config(redis_config): - host = _require_config_value(redis_config, "redis", "host") - if not _is_non_empty_string(host): - raise FQException("Invalid config: redis.host must be a non-empty string") - - port = _require_config_value(redis_config, "redis", "port") - if not _is_int_not_bool(port): - raise FQException("Invalid config: redis.port must be an integer") - - if "clustered" in redis_config and not isinstance(redis_config["clustered"], bool): - raise FQException("Invalid config: redis.clustered must be a boolean") - - -def _validate_optional_redis_config(redis_config): - if "password" in redis_config and redis_config["password"] is not None: - if not isinstance(redis_config["password"], str): - raise FQException("Invalid config: redis.password must be a string") - - -def _validate_identifier(identifier, message): - if not is_valid_identifier(identifier): - raise BadArgumentException(message) - - -def load_lua_scripts(instance, redis_client): - lua_script_path = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "scripts", "lua" - ) - - for script_name in LUA_SCRIPT_NAMES: - with open( - os.path.join(lua_script_path, "%s.lua" % script_name), - "r", - encoding="utf-8", - ) as script_file: - script = script_file.read() - setattr(instance, "_lua_%s_script" % script_name, script) - setattr( - instance, - "_lua_%s" % script_name, - redis_client.register_script(script), - ) - - -def validate_enqueue_arguments( - payload, - interval, - job_id, - queue_id, - queue_type, - requeue_limit, - default_requeue_limit, -): - if not is_valid_interval(interval): - raise BadArgumentException(INVALID_INTERVAL) - - _validate_identifier(job_id, INVALID_JOB_ID) - _validate_identifier(queue_id, INVALID_QUEUE_ID) - _validate_identifier(queue_type, INVALID_QUEUE_TYPE) - - if requeue_limit is None: - requeue_limit = default_requeue_limit - - if not is_valid_requeue_limit(requeue_limit): - raise BadArgumentException(INVALID_REQUEUE_LIMIT) - - try: - serialized_payload = serialize_payload(payload) - except TypeError: - raise BadArgumentException("can not serialize.") - - return serialized_payload, requeue_limit - - -def validate_dequeue_arguments(queue_type): - _validate_identifier(queue_type, INVALID_QUEUE_TYPE) - - -def validate_finish_arguments(job_id, queue_id, queue_type): - _validate_identifier(job_id, INVALID_JOB_ID) - _validate_identifier(queue_id, INVALID_QUEUE_ID) - _validate_identifier(queue_type, INVALID_QUEUE_TYPE) - - -def validate_interval_arguments(interval, queue_id, queue_type): - if not is_valid_interval(interval): - raise BadArgumentException(INVALID_INTERVAL) - - _validate_identifier(queue_id, INVALID_QUEUE_ID) - _validate_identifier(queue_type, INVALID_QUEUE_TYPE) - - -def validate_metrics_arguments(queue_type, queue_id): - if queue_id is not None and not is_valid_identifier(queue_id): - raise BadArgumentException(INVALID_QUEUE_ID) - - if queue_type is not None and not is_valid_identifier(queue_type): - raise BadArgumentException(INVALID_QUEUE_TYPE) - - -def validate_clear_queue_arguments(queue_type, queue_id): - if queue_id is None or not is_valid_identifier(queue_id): - raise BadArgumentException(INVALID_QUEUE_ID) - - if queue_type is None or not is_valid_identifier(queue_type): - raise BadArgumentException(INVALID_QUEUE_TYPE) - - -def validate_get_queue_length_arguments(queue_type, queue_id): - _validate_identifier(queue_type, INVALID_QUEUE_TYPE) - _validate_identifier(queue_id, INVALID_QUEUE_ID) - - -def decode_redis_value(value): - if isinstance(value, bytes): - return value.decode("utf-8") - return value - - -def format_dequeue_response(dequeue_response): - if len(dequeue_response) < 4: - return {"status": "failure"} - - queue_id, job_id, payload, requeues_remaining = dequeue_response - - if payload is None: - return {"status": "failure"} - - payload = deserialize_payload(payload) - - return { - "status": "success", - "queue_id": decode_redis_value(queue_id), - "job_id": decode_redis_value(job_id), - "payload": payload, - "requeues_remaining": int(requeues_remaining), - } - - -def format_metrics_counts(enqueue_details, dequeue_details): - enqueue_counts = {} - dequeue_counts = {} - for i in range(0, len(enqueue_details), 2): - enqueue_counts[str(decode_redis_value(enqueue_details[i]))] = int( - enqueue_details[i + 1] or 0 - ) - dequeue_counts[str(decode_redis_value(dequeue_details[i]))] = int( - dequeue_details[i + 1] or 0 - ) - return enqueue_counts, dequeue_counts - - -def format_queue_types(active_queue_types, ready_queue_types): - return convert_to_str(active_queue_types | ready_queue_types) - - -def format_queue_ids(ready_queues, active_queues): - active_queues = [decode_redis_value(i).split(":")[0] for i in active_queues] - all_queue_set = set(ready_queues) | set(active_queues) - return convert_to_str(all_queue_set) - - -def enqueue_script_args( - key_prefix, - queue_type, - queue_id, - job_id, - serialized_payload, - interval, - requeue_limit, -): - timestamp = str(generate_epoch()) - keys = [key_prefix, queue_type] - args = [ - timestamp, - queue_id, - job_id, - serialized_payload, - interval, - requeue_limit, - ] - return keys, args diff --git a/src/fq/keys.py b/src/fq/keys.py new file mode 100644 index 0000000..ddf3279 --- /dev/null +++ b/src/fq/keys.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class RedisKeys: + key_prefix: str + + @property + def active_queue_types(self): + return "%s:active:queue_type" % self.key_prefix + + @property + def ready_queue_types(self): + return "%s:ready:queue_type" % self.key_prefix + + @property + def interval_hash(self): + return "%s:interval" % self.key_prefix + + @property + def payload_hash(self): + return "%s:payload" % self.key_prefix + + @property + def deep_status(self): + return "fq:deep_status:%s" % self.key_prefix + + def ready_queue_set(self, queue_type): + return "%s:%s" % (self.key_prefix, queue_type) + + def active_queue_set(self, queue_type): + return "%s:%s:active" % (self.key_prefix, queue_type) + + def job_queue(self, queue_type, queue_id): + return "%s:%s:%s" % (self.key_prefix, queue_type, queue_id) + + def interval_member(self, queue_type, queue_id): + return "%s:%s" % (queue_type, queue_id) + + def payload_member(self, queue_type, queue_id, job_id): + return "%s:%s:%s" % (queue_type, queue_id, job_id) diff --git a/src/fq/lua.py b/src/fq/lua.py new file mode 100644 index 0000000..c05442b --- /dev/null +++ b/src/fq/lua.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from dataclasses import dataclass +from pathlib import Path + + +LUA_SCRIPT_NAMES = ("enqueue", "dequeue", "finish", "interval", "requeue", "metrics") + + +@dataclass(frozen=True) +class Lua: + enqueue: object + dequeue: object + finish: object + interval: object + requeue: object + metrics: object + + @classmethod + def register(cls, redis_client): + registered_scripts = { + script_name: redis_client.register_script(cls._read_script(script_name)) + for script_name in LUA_SCRIPT_NAMES + } + return cls(**registered_scripts) + + @staticmethod + def _read_script(script_name): + script_path = ( + Path(__file__).with_name("scripts") / "lua" / ("%s.lua" % script_name) + ) + return script_path.read_text(encoding="utf-8") diff --git a/src/fq/queue.py b/src/fq/queue.py index 52b2658..147ae92 100644 --- a/src/fq/queue.py +++ b/src/fq/queue.py @@ -4,18 +4,20 @@ import asyncio -from redis.asyncio import Redis -from redis.asyncio.cluster import RedisCluster - -from fq.core import ( - enqueue_script_args, +from fq.config import FQConfig +from fq.exceptions import BadArgumentException +from fq.keys import RedisKeys +from fq.lua import Lua +from fq.redis import create_async_redis_client, validate_async_redis_connection +from fq.responses import ( + decode_redis_value, format_dequeue_response, format_metrics_counts, format_queue_ids, format_queue_types, - generate_epoch, - load_lua_scripts, - normalize_config, +) +from fq.utils import generate_epoch +from fq.validators import ( validate_clear_queue_arguments, validate_dequeue_arguments, validate_enqueue_arguments, @@ -24,100 +26,38 @@ validate_interval_arguments, validate_metrics_arguments, ) -from fq.exceptions import FQException, BadArgumentException class FQ(object): - """The FQ object is the core of this queue. - FQ does the following. - - 1. Accepts structured configuration. - 2. Initializes the queue. - 3. Exposes functions to interact with the queue. - """ + """Async Flowdacity Queue API.""" def __init__(self, config): - """Construct a FQ object by doing the following. - 1. Store the queue configuration. - 2. Validate the config shape. - """ - self._r = None # redis client placeholder - self.config = normalize_config(config) + self._r = None + self._scripts = None + self.config = FQConfig.from_mapping(config) + self._keys = RedisKeys(self.config.redis.key_prefix) + + self._key_prefix = self.config.redis.key_prefix + self._job_expire_interval = int(self.config.job_expire_interval) + self._default_job_requeue_limit = int( + self.config.default_job_requeue_limit + ) async def initialize(self): - """Async initializer to set up redis and lua scripts.""" - fq_config = self.config["fq"] - redis_config = self.config["redis"] - - self._key_prefix = redis_config["key_prefix"] - self._job_expire_interval = int(fq_config["job_expire_interval"]) - self._default_job_requeue_limit = int(fq_config["default_job_requeue_limit"]) - - redis_connection_type = redis_config["conn_type"] - db = redis_config["db"] - - if redis_connection_type == "unix_sock": - self._r = Redis( - db=db, - unix_socket_path=redis_config["unix_socket_path"], - ) - elif redis_connection_type == "tcp_sock": - isclustered = False - if "clustered" in redis_config: - isclustered = redis_config["clustered"] - - if isclustered: - startup_nodes = [ - { - "host": redis_config["host"], - "port": int(redis_config["port"]), - } - ] - self._r = RedisCluster( - startup_nodes=startup_nodes, - decode_responses=False, - socket_timeout=5, - ) - else: - self._r = Redis( - db=db, - host=redis_config["host"], - port=int(redis_config["port"]), - password=redis_config.get("password"), - ) - else: - raise FQException("Unknown redis conn_type: %s" % redis_connection_type) - - await self._validate_redis_connection() - self._load_lua_scripts() - - async def _validate_redis_connection(self): - """Ping redis once to surface bad connection details early.""" - if self._r is None: - raise FQException("Redis client is not initialized") - - ping = getattr(self._r, "ping", None) - if not callable(ping): - return - - try: - result = await ping() - except Exception as exc: - raise FQException("Failed to connect to Redis: %s" % exc) from exc - - if result is False: - raise FQException("Failed to connect to Redis: ping returned False") + """Set up the async Redis client and register Lua scripts.""" + self._r = create_async_redis_client(self.config.redis) + await validate_async_redis_connection(self._r) + self._register_lua_scripts() def redis_client(self): return self._r - def _load_lua_scripts(self): - """Loads all lua scripts required by FQ.""" - load_lua_scripts(self, self._r) + def _register_lua_scripts(self): + self._scripts = Lua.register(self._r) def reload_lua_scripts(self): - """Lets user reload the lua scripts in run time.""" - self._load_lua_scripts() + """Lets user reload the Lua scripts at run time.""" + self._register_lua_scripts() async def enqueue( self, @@ -128,10 +68,8 @@ async def enqueue( queue_type="default", requeue_limit=None, ): - """Enqueues the job into the specified queue_id - of a particular queue_type - """ - serialized_payload, requeue_limit = validate_enqueue_arguments( + """Enqueue a job into the specified queue_id and queue_type.""" + enqueue_args = validate_enqueue_arguments( payload, interval, job_id, @@ -140,131 +78,94 @@ async def enqueue( requeue_limit, self._default_job_requeue_limit, ) - keys, args = enqueue_script_args( - self._key_prefix, - queue_type, + + keys = [self._key_prefix, queue_type] + args = [ + str(generate_epoch()), queue_id, job_id, - serialized_payload, + enqueue_args.serialized_payload, interval, - requeue_limit, - ) - await self._lua_enqueue(keys=keys, args=args) + enqueue_args.requeue_limit, + ] + await self._scripts.enqueue(keys=keys, args=args) return {"status": "queued"} async def dequeue(self, queue_type="default"): - """Dequeues a job from any of the ready queues - based on the queue_type. If no job is ready, - returns a failure status. - """ + """Dequeue a ready job for queue_type, or return failure.""" validate_dequeue_arguments(queue_type) - timestamp = str(generate_epoch()) - keys = [self._key_prefix, queue_type] - args = [timestamp, self._job_expire_interval] + args = [str(generate_epoch()), self._job_expire_interval] - dequeue_response = await self._lua_dequeue(keys=keys, args=args) + dequeue_response = await self._scripts.dequeue(keys=keys, args=args) return format_dequeue_response(dequeue_response) async def finish(self, job_id, queue_id, queue_type="default"): - """Marks any dequeued job as *completed successfully*. - Any job which gets a finish will be treated as complete - and will be removed from the FQ. - """ + """Mark a dequeued job as completed successfully.""" validate_finish_arguments(job_id, queue_id, queue_type) keys = [self._key_prefix, queue_type] - args = [queue_id, job_id] - finish_response = await self._lua_finish(keys=keys, args=args) + finish_response = await self._scripts.finish(keys=keys, args=args) if finish_response == 0: - # the finish failed. return {"status": "failure"} return {"status": "success"} async def interval(self, interval, queue_id, queue_type="default"): - """Updates the interval for a specific queue_id - of a particular queue type. - """ + """Update the interval for a queue_id and queue_type.""" validate_interval_arguments(interval, queue_id, queue_type) - # generate the interval key - interval_hmap_key = "%s:interval" % self._key_prefix - interval_queue_key = "%s:%s" % (queue_type, queue_id) - keys = [interval_hmap_key, interval_queue_key] - + keys = [ + self._keys.interval_hash, + self._keys.interval_member(queue_type, queue_id), + ] args = [interval] - interval_response = await self._lua_interval(keys=keys, args=args) + + interval_response = await self._scripts.interval(keys=keys, args=args) if interval_response == 0: - # the queue with the id and type does not exist. return {"status": "failure"} - else: - return {"status": "success"} + + return {"status": "success"} async def requeue(self): - """Re-queues any expired job (one which does not get an expire - before the job_expiry_interval) back into their respective queue. - This function has to be run at specified intervals to ensure the - expired jobs are re-queued back. - """ + """Re-queue expired active jobs back into their ready queues.""" timestamp = str(generate_epoch()) - # get all queue_types and requeue one by one. - # not recommended to do this entire process - # in lua as it might take long and block other - # enqueues and dequeues. - active_queue_type_list = await self._r.smembers( - "%s:active:queue_type" % self._key_prefix - ) + active_queue_type_list = await self._r.smembers(self._keys.active_queue_types) for queue_type in active_queue_type_list: - # requeue all expired jobs in all queue types. - - queue_type = queue_type.decode("utf-8") - + queue_type = decode_redis_value(queue_type) keys = [self._key_prefix, queue_type] - args = [timestamp] - job_discard_list = await self._lua_requeue(keys=keys, args=args) - # discard the jobs if any + job_discard_list = await self._scripts.requeue(keys=keys, args=args) for job in job_discard_list: - queue_id, job_id = job.decode("utf-8").split(":") - # explicitly finishing a job - # is nothing but discard. + queue_id, job_id = decode_redis_value(job).split(":") await self.finish( - job_id=job_id, queue_id=queue_id, queue_type=queue_type + job_id=job_id, + queue_id=queue_id, + queue_type=queue_type, ) async def metrics(self, queue_type=None, queue_id=None): - """Provides a way to get statistics about various parameters like, - * global enqueue / dequeue rates per min. - * per queue enqueue / dequeue rates per min. - * queue length of each queue. - * list of queue ids for each queue type. - """ + """Return global, queue-type, or queue-specific metrics.""" validate_metrics_arguments(queue_type, queue_id) response = {"status": "failure"} if not queue_type and not queue_id: - # return global stats. - # list of active queue types (ready + active) - active_queue_types = await self._r.smembers( - "%s:active:queue_type" % self._key_prefix - ) - ready_queue_types = await self._r.smembers( - "%s:ready:queue_type" % self._key_prefix - ) + active_queue_types = await self._r.smembers(self._keys.active_queue_types) + ready_queue_types = await self._r.smembers(self._keys.ready_queue_types) queue_types = format_queue_types(active_queue_types, ready_queue_types) - # global rates for past 10 minutes - timestamp = str(generate_epoch()) + keys = [self._key_prefix] - args = [timestamp] - enqueue_details, dequeue_details = await self._lua_metrics( - keys=keys, args=args + args = [str(generate_epoch())] + enqueue_details, dequeue_details = await self._scripts.metrics( + keys=keys, + args=args, ) enqueue_counts, dequeue_counts = format_metrics_counts( - enqueue_details, dequeue_details + enqueue_details, + dequeue_details, ) response.update( @@ -276,35 +177,28 @@ async def metrics(self, queue_type=None, queue_id=None): } ) return response - elif queue_type and not queue_id: - # return list of queue_ids. - # get data from two sorted sets in a transaction + + if queue_type and not queue_id: pipe = self._r.pipeline() - pipe.zrange("%s:%s" % (self._key_prefix, queue_type), 0, -1) - pipe.zrange("%s:%s:active" % (self._key_prefix, queue_type), 0, -1) + pipe.zrange(self._keys.ready_queue_set(queue_type), 0, -1) + pipe.zrange(self._keys.active_queue_set(queue_type), 0, -1) ready_queues, active_queues = await pipe.execute() - # extract the queue_ids from the queue_id:job_id string queue_list = format_queue_ids(ready_queues, active_queues) response.update({"status": "success", "queue_ids": queue_list}) return response - elif queue_type and queue_id: - # return specific details. - # queue specific rates for past 10 minutes - timestamp = str(generate_epoch()) - keys = ["%s:%s:%s" % (self._key_prefix, queue_type, queue_id)] - args = [timestamp] - enqueue_details, dequeue_details = await self._lua_metrics( - keys=keys, args=args - ) - enqueue_counts, dequeue_counts = format_metrics_counts( - enqueue_details, dequeue_details + if queue_type and queue_id: + keys = [self._keys.job_queue(queue_type, queue_id)] + args = [str(generate_epoch())] + enqueue_details, dequeue_details = await self._scripts.metrics( + keys=keys, + args=args, ) - - # get the queue length for the job queue - queue_length = await self._r.llen( - "%s:%s:%s" % (self._key_prefix, queue_type, queue_id) + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, + dequeue_details, ) + queue_length = await self._r.llen(self._keys.job_queue(queue_type, queue_id)) response.update( { @@ -315,7 +209,8 @@ async def metrics(self, queue_type=None, queue_id=None): } ) return response - elif not queue_type and queue_id: + + if not queue_type and queue_id: raise BadArgumentException( "`queue_id` should be accompanied by `queue_type`." ) @@ -324,24 +219,17 @@ async def metrics(self, queue_type=None, queue_id=None): async def deep_status(self): """ - To check the availability of redis. If redis is down get will throw exception + Check Redis availability. If Redis is down, set() will raise. :return: value or None """ - return await self._r.set( - "fq:deep_status:{}".format(self._key_prefix), "sharq_deep_status" - ) + return await self._r.set(self._keys.deep_status, "sharq_deep_status") async def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): - """clear the all entries in queue with particular queue_id - and queue_type. It takes an optional argument, - purge_all : if True, then it will remove the related resources - from the redis. - """ + """Clear entries in a queue and optionally purge related resources.""" validate_clear_queue_arguments(queue_type, queue_id) response = {"status": "Failure", "message": "No queued calls found"} - # remove from the primary sorted set - primary_set = "{}:{}".format(self._key_prefix, queue_type) + primary_set = self._keys.ready_queue_set(queue_type) queued_status = await self._r.zrem(primary_set, queue_id) if queued_status: response.update( @@ -350,30 +238,24 @@ async def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): "message": "Successfully removed all queued calls", } ) - # do a full cleanup of reources - # although this is not necessary as we don't remove resources - # while dequeue operation - job_queue_list = "{}:{}:{}".format(self._key_prefix, queue_type, queue_id) + + job_queue_list = self._keys.job_queue(queue_type, queue_id) if queued_status and purge_all: job_list = await self._r.lrange(job_queue_list, 0, -1) pipe = self._r.pipeline() - # clear the payload data for job_uuid for job_uuid in job_list: if job_uuid is None: continue - if isinstance(job_uuid, bytes): - job_uuid_str = job_uuid.decode("utf-8") - else: - job_uuid_str = job_uuid - payload_set = "{}:payload".format(self._key_prefix) - job_payload_key = "{}:{}:{}".format(queue_type, queue_id, job_uuid_str) - pipe.hdel(payload_set, job_payload_key) - - # clear job request interval - interval_set = "{}:interval".format(self._key_prefix) - job_interval_key = "{}:{}".format(queue_type, queue_id) - pipe.hdel(interval_set, job_interval_key) - # clear job_queue_list + job_uuid_str = decode_redis_value(job_uuid) + pipe.hdel( + self._keys.payload_hash, + self._keys.payload_member(queue_type, queue_id, job_uuid_str), + ) + + pipe.hdel( + self._keys.interval_hash, + self._keys.interval_member(queue_type, queue_id), + ) pipe.delete(job_queue_list) await pipe.execute() response.update( @@ -383,42 +265,30 @@ async def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): } ) else: - # always delete the job queue list await self._r.delete(job_queue_list) + return response async def get_queue_length(self, queue_type, queue_id): """ - Return the current length present in redis key of type list - Redis key structure : key_prefix : queue_type : queue_id + Return the current Redis list length for key_prefix:queue_type:queue_id. """ - validate_get_queue_length_arguments(queue_type, queue_id) - - redis_key = self._key_prefix + ":" + queue_type + ":" + queue_id - current_queue_length = await self._r.llen(redis_key) - return current_queue_length + return await self._r.llen(self._keys.job_queue(queue_type, queue_id)) async def close(self): - """ - Cleanly close the underlying Redis client / connection pool. - - This is intended to be called by tests or by application shutdown - hooks to avoid ResourceWarning: unclosed . - """ + """Cleanly close the underlying Redis client or connection pool.""" if self._r is None: return conn = self._r - # Prefer the asyncio-style aclose() if available (redis-py >= 4.2+) aclose = getattr(conn, "aclose", None) if callable(aclose): await aclose() self._r = None return - # Older / alternate API: close() [+ wait_closed()] close = getattr(conn, "close", None) if callable(close): maybe_coro = close() @@ -431,7 +301,6 @@ async def close(self): if asyncio.iscoroutine(maybe_coro): await maybe_coro - # As a final fallback, disconnect the connection pool pool = getattr(conn, "connection_pool", None) if pool is not None: disconnect = getattr(pool, "disconnect", None) diff --git a/src/fq/redis.py b/src/fq/redis.py new file mode 100644 index 0000000..9eb1f28 --- /dev/null +++ b/src/fq/redis.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from redis import Redis as SyncRedis +from redis import RedisCluster as SyncRedisCluster +from redis.asyncio import Redis as AsyncRedis +from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster + +from fq.exceptions import FQException + + +def create_async_redis_client(redis_config): + if redis_config.conn_type == "unix_sock": + return AsyncRedis( + db=redis_config.db, + unix_socket_path=redis_config.unix_socket_path, + ) + + if redis_config.conn_type == "tcp_sock": + if redis_config.clustered: + startup_nodes = [ + { + "host": redis_config.host, + "port": int(redis_config.port), + } + ] + return AsyncRedisCluster( + startup_nodes=startup_nodes, + decode_responses=False, + socket_timeout=5, + ) + + return AsyncRedis( + db=redis_config.db, + host=redis_config.host, + port=int(redis_config.port), + password=redis_config.password, + ) + + raise FQException("Unknown redis conn_type: %s" % redis_config.conn_type) + + +def create_sync_redis_client(redis_config): + if redis_config.conn_type == "unix_sock": + return SyncRedis( + db=redis_config.db, + unix_socket_path=redis_config.unix_socket_path, + ) + + if redis_config.conn_type == "tcp_sock": + if redis_config.clustered: + return SyncRedisCluster( + host=redis_config.host, + port=int(redis_config.port), + decode_responses=False, + socket_timeout=5, + ) + + return SyncRedis( + db=redis_config.db, + host=redis_config.host, + port=int(redis_config.port), + password=redis_config.password, + ) + + raise FQException("Unknown redis conn_type: %s" % redis_config.conn_type) + + +async def validate_async_redis_connection(redis_client): + if redis_client is None: + raise FQException("Redis client is not initialized") + + ping = getattr(redis_client, "ping", None) + if not callable(ping): + return + + try: + result = await ping() + except Exception as exc: + raise FQException("Failed to connect to Redis: %s" % exc) from exc + + if result is False: + raise FQException("Failed to connect to Redis: ping returned False") + + +def validate_sync_redis_connection(redis_client): + if redis_client is None: + raise FQException("Redis client is not initialized") + + ping = getattr(redis_client, "ping", None) + if not callable(ping): + return + + try: + result = ping() + except Exception as exc: + raise FQException("Failed to connect to Redis: %s" % exc) from exc + + if result is False: + raise FQException("Failed to connect to Redis: ping returned False") diff --git a/src/fq/responses.py b/src/fq/responses.py new file mode 100644 index 0000000..04092ed --- /dev/null +++ b/src/fq/responses.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from fq.utils import convert_to_str, deserialize_payload + + +def decode_redis_value(value): + if isinstance(value, bytes): + return value.decode("utf-8") + return value + + +def format_dequeue_response(dequeue_response): + if len(dequeue_response) < 4: + return {"status": "failure"} + + queue_id, job_id, payload, requeues_remaining = dequeue_response + + if payload is None: + return {"status": "failure"} + + return { + "status": "success", + "queue_id": decode_redis_value(queue_id), + "job_id": decode_redis_value(job_id), + "payload": deserialize_payload(payload), + "requeues_remaining": int(requeues_remaining), + } + + +def format_metrics_counts(enqueue_details, dequeue_details): + enqueue_counts = {} + dequeue_counts = {} + for i in range(0, len(enqueue_details), 2): + enqueue_counts[str(decode_redis_value(enqueue_details[i]))] = int( + enqueue_details[i + 1] or 0 + ) + dequeue_counts[str(decode_redis_value(dequeue_details[i]))] = int( + dequeue_details[i + 1] or 0 + ) + return enqueue_counts, dequeue_counts + + +def format_queue_types(active_queue_types, ready_queue_types): + return convert_to_str(set(active_queue_types) | set(ready_queue_types)) + + +def format_queue_ids(ready_queues, active_queues): + active_queues = [decode_redis_value(i).split(":")[0] for i in active_queues] + all_queue_set = set(ready_queues) | set(active_queues) + return convert_to_str(all_queue_set) diff --git a/src/fq/sync/queue.py b/src/fq/sync/queue.py index c8444e4..e3a749e 100644 --- a/src/fq/sync/queue.py +++ b/src/fq/sync/queue.py @@ -1,18 +1,20 @@ # -*- coding: utf-8 -*- # Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. -from redis import Redis, RedisCluster - -from fq.core import ( +from fq.config import FQConfig +from fq.exceptions import BadArgumentException +from fq.keys import RedisKeys +from fq.lua import Lua +from fq.redis import create_sync_redis_client, validate_sync_redis_connection +from fq.responses import ( decode_redis_value, - enqueue_script_args, format_dequeue_response, format_metrics_counts, format_queue_ids, format_queue_types, - generate_epoch, - load_lua_scripts, - normalize_config, +) +from fq.utils import generate_epoch +from fq.validators import ( validate_clear_queue_arguments, validate_dequeue_arguments, validate_enqueue_arguments, @@ -21,85 +23,38 @@ validate_interval_arguments, validate_metrics_arguments, ) -from fq.exceptions import BadArgumentException, FQException class FQ(object): - """Synchronous FQ API backed by redis-py's synchronous client.""" + """Synchronous Flowdacity Queue API.""" def __init__(self, config): self._r = None - self.config = normalize_config(config) + self._scripts = None + self.config = FQConfig.from_mapping(config) + self._keys = RedisKeys(self.config.redis.key_prefix) + + self._key_prefix = self.config.redis.key_prefix + self._job_expire_interval = int(self.config.job_expire_interval) + self._default_job_requeue_limit = int( + self.config.default_job_requeue_limit + ) def initialize(self): - """Set up the synchronous Redis client and Lua scripts.""" - fq_config = self.config["fq"] - redis_config = self.config["redis"] - - self._key_prefix = redis_config["key_prefix"] - self._job_expire_interval = int(fq_config["job_expire_interval"]) - self._default_job_requeue_limit = int(fq_config["default_job_requeue_limit"]) - - redis_connection_type = redis_config["conn_type"] - db = redis_config["db"] - - if redis_connection_type == "unix_sock": - self._r = Redis( - db=db, - unix_socket_path=redis_config["unix_socket_path"], - ) - elif redis_connection_type == "tcp_sock": - isclustered = False - if "clustered" in redis_config: - isclustered = redis_config["clustered"] - - if isclustered: - self._r = RedisCluster( - host=redis_config["host"], - port=int(redis_config["port"]), - decode_responses=False, - socket_timeout=5, - ) - else: - self._r = Redis( - db=db, - host=redis_config["host"], - port=int(redis_config["port"]), - password=redis_config.get("password"), - ) - else: - raise FQException("Unknown redis conn_type: %s" % redis_connection_type) - - self._validate_redis_connection() - self._load_lua_scripts() - - def _validate_redis_connection(self): - """Ping Redis once to surface bad connection details early.""" - if self._r is None: - raise FQException("Redis client is not initialized") - - ping = getattr(self._r, "ping", None) - if not callable(ping): - return - - try: - result = ping() - except Exception as exc: - raise FQException("Failed to connect to Redis: %s" % exc) from exc - - if result is False: - raise FQException("Failed to connect to Redis: ping returned False") + """Set up the synchronous Redis client and register Lua scripts.""" + self._r = create_sync_redis_client(self.config.redis) + validate_sync_redis_connection(self._r) + self._register_lua_scripts() def redis_client(self): return self._r - def _load_lua_scripts(self): - """Loads all Lua scripts required by FQ.""" - load_lua_scripts(self, self._r) + def _register_lua_scripts(self): + self._scripts = Lua.register(self._r) def reload_lua_scripts(self): """Lets user reload the Lua scripts at run time.""" - self._load_lua_scripts() + self._register_lua_scripts() def enqueue( self, @@ -111,7 +66,7 @@ def enqueue( requeue_limit=None, ): """Enqueue a job into the specified queue_id and queue_type.""" - serialized_payload, requeue_limit = validate_enqueue_arguments( + enqueue_args = validate_enqueue_arguments( payload, interval, job_id, @@ -120,27 +75,27 @@ def enqueue( requeue_limit, self._default_job_requeue_limit, ) - keys, args = enqueue_script_args( - self._key_prefix, - queue_type, + + keys = [self._key_prefix, queue_type] + args = [ + str(generate_epoch()), queue_id, job_id, - serialized_payload, + enqueue_args.serialized_payload, interval, - requeue_limit, - ) - self._lua_enqueue(keys=keys, args=args) + enqueue_args.requeue_limit, + ] + self._scripts.enqueue(keys=keys, args=args) return {"status": "queued"} def dequeue(self, queue_type="default"): - """Dequeue a ready job for the queue_type, or return failure.""" + """Dequeue a ready job for queue_type, or return failure.""" validate_dequeue_arguments(queue_type) - timestamp = str(generate_epoch()) keys = [self._key_prefix, queue_type] - args = [timestamp, self._job_expire_interval] + args = [str(generate_epoch()), self._job_expire_interval] - dequeue_response = self._lua_dequeue(keys=keys, args=args) + dequeue_response = self._scripts.dequeue(keys=keys, args=args) return format_dequeue_response(dequeue_response) def finish(self, job_id, queue_id, queue_type="default"): @@ -150,7 +105,7 @@ def finish(self, job_id, queue_id, queue_type="default"): keys = [self._key_prefix, queue_type] args = [queue_id, job_id] - finish_response = self._lua_finish(keys=keys, args=args) + finish_response = self._scripts.finish(keys=keys, args=args) if finish_response == 0: return {"status": "failure"} @@ -160,27 +115,27 @@ def interval(self, interval, queue_id, queue_type="default"): """Update the interval for a queue_id and queue_type.""" validate_interval_arguments(interval, queue_id, queue_type) - interval_hmap_key = "%s:interval" % self._key_prefix - interval_queue_key = "%s:%s" % (queue_type, queue_id) - keys = [interval_hmap_key, interval_queue_key] + keys = [ + self._keys.interval_hash, + self._keys.interval_member(queue_type, queue_id), + ] args = [interval] - interval_response = self._lua_interval(keys=keys, args=args) + interval_response = self._scripts.interval(keys=keys, args=args) if interval_response == 0: return {"status": "failure"} + return {"status": "success"} def requeue(self): """Re-queue expired active jobs back into their ready queues.""" timestamp = str(generate_epoch()) - active_queue_type_list = self._r.smembers( - "%s:active:queue_type" % self._key_prefix - ) + active_queue_type_list = self._r.smembers(self._keys.active_queue_types) for queue_type in active_queue_type_list: queue_type = decode_redis_value(queue_type) keys = [self._key_prefix, queue_type] args = [timestamp] - job_discard_list = self._lua_requeue(keys=keys, args=args) + job_discard_list = self._scripts.requeue(keys=keys, args=args) for job in job_discard_list: queue_id, job_id = decode_redis_value(job).split(":") self.finish(job_id=job_id, queue_id=queue_id, queue_type=queue_type) @@ -191,20 +146,19 @@ def metrics(self, queue_type=None, queue_id=None): response = {"status": "failure"} if not queue_type and not queue_id: - active_queue_types = self._r.smembers( - "%s:active:queue_type" % self._key_prefix - ) - ready_queue_types = self._r.smembers( - "%s:ready:queue_type" % self._key_prefix - ) + active_queue_types = self._r.smembers(self._keys.active_queue_types) + ready_queue_types = self._r.smembers(self._keys.ready_queue_types) queue_types = format_queue_types(active_queue_types, ready_queue_types) - timestamp = str(generate_epoch()) keys = [self._key_prefix] - args = [timestamp] - enqueue_details, dequeue_details = self._lua_metrics(keys=keys, args=args) + args = [str(generate_epoch())] + enqueue_details, dequeue_details = self._scripts.metrics( + keys=keys, + args=args, + ) enqueue_counts, dequeue_counts = format_metrics_counts( - enqueue_details, dequeue_details + enqueue_details, + dequeue_details, ) response.update( @@ -216,26 +170,28 @@ def metrics(self, queue_type=None, queue_id=None): } ) return response - elif queue_type and not queue_id: + + if queue_type and not queue_id: pipe = self._r.pipeline() - pipe.zrange("%s:%s" % (self._key_prefix, queue_type), 0, -1) - pipe.zrange("%s:%s:active" % (self._key_prefix, queue_type), 0, -1) + pipe.zrange(self._keys.ready_queue_set(queue_type), 0, -1) + pipe.zrange(self._keys.active_queue_set(queue_type), 0, -1) ready_queues, active_queues = pipe.execute() queue_list = format_queue_ids(ready_queues, active_queues) response.update({"status": "success", "queue_ids": queue_list}) return response - elif queue_type and queue_id: - timestamp = str(generate_epoch()) - keys = ["%s:%s:%s" % (self._key_prefix, queue_type, queue_id)] - args = [timestamp] - enqueue_details, dequeue_details = self._lua_metrics(keys=keys, args=args) - enqueue_counts, dequeue_counts = format_metrics_counts( - enqueue_details, dequeue_details - ) - queue_length = self._r.llen( - "%s:%s:%s" % (self._key_prefix, queue_type, queue_id) + if queue_type and queue_id: + keys = [self._keys.job_queue(queue_type, queue_id)] + args = [str(generate_epoch())] + enqueue_details, dequeue_details = self._scripts.metrics( + keys=keys, + args=args, ) + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, + dequeue_details, + ) + queue_length = self._r.llen(self._keys.job_queue(queue_type, queue_id)) response.update( { @@ -246,7 +202,8 @@ def metrics(self, queue_type=None, queue_id=None): } ) return response - elif not queue_type and queue_id: + + if not queue_type and queue_id: raise BadArgumentException( "`queue_id` should be accompanied by `queue_type`." ) @@ -258,16 +215,14 @@ def deep_status(self): Check Redis availability. If Redis is down, set() will raise. :return: value or None """ - return self._r.set( - "fq:deep_status:{}".format(self._key_prefix), "sharq_deep_status" - ) + return self._r.set(self._keys.deep_status, "sharq_deep_status") def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): """Clear entries in a queue and optionally purge related resources.""" validate_clear_queue_arguments(queue_type, queue_id) response = {"status": "Failure", "message": "No queued calls found"} - primary_set = "{}:{}".format(self._key_prefix, queue_type) + primary_set = self._keys.ready_queue_set(queue_type) queued_status = self._r.zrem(primary_set, queue_id) if queued_status: response.update( @@ -277,7 +232,7 @@ def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): } ) - job_queue_list = "{}:{}:{}".format(self._key_prefix, queue_type, queue_id) + job_queue_list = self._keys.job_queue(queue_type, queue_id) if queued_status and purge_all: job_list = self._r.lrange(job_queue_list, 0, -1) pipe = self._r.pipeline() @@ -285,13 +240,15 @@ def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): if job_uuid is None: continue job_uuid_str = decode_redis_value(job_uuid) - payload_set = "{}:payload".format(self._key_prefix) - job_payload_key = "{}:{}:{}".format(queue_type, queue_id, job_uuid_str) - pipe.hdel(payload_set, job_payload_key) + pipe.hdel( + self._keys.payload_hash, + self._keys.payload_member(queue_type, queue_id, job_uuid_str), + ) - interval_set = "{}:interval".format(self._key_prefix) - job_interval_key = "{}:{}".format(queue_type, queue_id) - pipe.hdel(interval_set, job_interval_key) + pipe.hdel( + self._keys.interval_hash, + self._keys.interval_member(queue_type, queue_id), + ) pipe.delete(job_queue_list) pipe.execute() response.update( @@ -302,6 +259,7 @@ def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): ) else: self._r.delete(job_queue_list) + return response def get_queue_length(self, queue_type, queue_id): @@ -309,9 +267,7 @@ def get_queue_length(self, queue_type, queue_id): Return the current Redis list length for key_prefix:queue_type:queue_id. """ validate_get_queue_length_arguments(queue_type, queue_id) - - redis_key = self._key_prefix + ":" + queue_type + ":" + queue_id - return self._r.llen(redis_key) + return self._r.llen(self._keys.job_queue(queue_type, queue_id)) def close(self): """Close the underlying synchronous Redis client.""" diff --git a/src/fq/validators.py b/src/fq/validators.py new file mode 100644 index 0000000..5e4913e --- /dev/null +++ b/src/fq/validators.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from dataclasses import dataclass + +from fq.exceptions import BadArgumentException +from fq.utils import ( + is_valid_identifier, + is_valid_interval, + is_valid_requeue_limit, + serialize_payload, +) + + +INVALID_INTERVAL = "`interval` has an invalid value." +INVALID_JOB_ID = "`job_id` has an invalid value." +INVALID_QUEUE_ID = "`queue_id` has an invalid value." +INVALID_QUEUE_TYPE = "`queue_type` has an invalid value." +INVALID_REQUEUE_LIMIT = "`requeue_limit` has an invalid value." + + +@dataclass(frozen=True) +class EnqueueArguments: + serialized_payload: bytes + requeue_limit: int + + +def validate_enqueue_arguments( + payload, + interval, + job_id, + queue_id, + queue_type, + requeue_limit, + default_requeue_limit, +): + if not is_valid_interval(interval): + raise BadArgumentException(INVALID_INTERVAL) + + _validate_identifier(job_id, INVALID_JOB_ID) + _validate_identifier(queue_id, INVALID_QUEUE_ID) + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) + + if requeue_limit is None: + requeue_limit = default_requeue_limit + + if not is_valid_requeue_limit(requeue_limit): + raise BadArgumentException(INVALID_REQUEUE_LIMIT) + + try: + serialized_payload = serialize_payload(payload) + except TypeError: + raise BadArgumentException("can not serialize.") + + return EnqueueArguments( + serialized_payload=serialized_payload, + requeue_limit=requeue_limit, + ) + + +def validate_dequeue_arguments(queue_type): + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) + + +def validate_finish_arguments(job_id, queue_id, queue_type): + _validate_identifier(job_id, INVALID_JOB_ID) + _validate_identifier(queue_id, INVALID_QUEUE_ID) + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) + + +def validate_interval_arguments(interval, queue_id, queue_type): + if not is_valid_interval(interval): + raise BadArgumentException(INVALID_INTERVAL) + + _validate_identifier(queue_id, INVALID_QUEUE_ID) + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) + + +def validate_metrics_arguments(queue_type, queue_id): + if queue_id is not None and not is_valid_identifier(queue_id): + raise BadArgumentException(INVALID_QUEUE_ID) + + if queue_type is not None and not is_valid_identifier(queue_type): + raise BadArgumentException(INVALID_QUEUE_TYPE) + + +def validate_clear_queue_arguments(queue_type, queue_id): + if queue_id is None or not is_valid_identifier(queue_id): + raise BadArgumentException(INVALID_QUEUE_ID) + + if queue_type is None or not is_valid_identifier(queue_type): + raise BadArgumentException(INVALID_QUEUE_TYPE) + + +def validate_get_queue_length_arguments(queue_type, queue_id): + _validate_identifier(queue_type, INVALID_QUEUE_TYPE) + _validate_identifier(queue_id, INVALID_QUEUE_ID) + + +def _validate_identifier(identifier, message): + if not is_valid_identifier(identifier): + raise BadArgumentException(message) diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index eef44f4..a48d38f 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -4,6 +4,7 @@ import unittest from contextlib import suppress +from types import SimpleNamespace from unittest.mock import patch from fq import FQ @@ -134,7 +135,7 @@ def test_invalid_config_type_raises(self): FQ("does-not-exist.conf") async def test_initialize_fails_fast_on_bad_redis(self): - with patch("fq.queue.Redis", FakeRedisConnectionFailure): + with patch("fq.redis.AsyncRedis", FakeRedisConnectionFailure): fq = FQ(self.config) with self.assertRaisesRegex(FQException, "Failed to connect to Redis"): await fq.initialize() @@ -144,7 +145,7 @@ async def test_cluster_initialization(self): config = build_test_config( redis={"key_prefix": "test_fq_cluster", "clustered": True} ) - with patch("fq.queue.RedisCluster", FakeCluster): + with patch("fq.redis.AsyncRedisCluster", FakeCluster): fq = FQ(config) await fq.initialize() self.assertIsInstance(fq.redis_client(), FakeCluster) @@ -177,7 +178,7 @@ async def test_dequeue_payload_none(self): self.fq_instance = fq await fq.initialize() fake_dequeue = FakeLuaDequeue() - fq._lua_dequeue = fake_dequeue + fq._scripts = SimpleNamespace(dequeue=fake_dequeue) result = await fq.dequeue() self.assertEqual(result["status"], "failure") self.assertTrue(fake_dequeue.called) @@ -201,12 +202,14 @@ async def test_close_fallback_paths(self): async def test_deep_status_calls_set(self): """Covers deep_status (queue.py line 420).""" fq = FQ(self.config) - fq._key_prefix = fq.config["redis"]["key_prefix"] fq._r = FakeRedisForDeepStatus() await fq.deep_status() self.assertEqual( fq._r.key_set, - ("fq:deep_status:{}".format(fq._key_prefix), "sharq_deep_status"), + ( + "fq:deep_status:{}".format(fq.config.redis.key_prefix), + "sharq_deep_status", + ), ) def test_is_valid_identifier_non_string(self): @@ -218,7 +221,6 @@ def test_is_valid_identifier_non_string(self): async def test_clear_queue_purge_all_with_mixed_job_ids(self): """Covers purge_all loop branches (queue.py lines 463-468, 474-479).""" fq = FQ(self.config) - fq._key_prefix = fq.config["redis"]["key_prefix"] fq._r = FakeRedisForClear() response = await fq.clear_queue("qt", "qid", purge_all=True) self.assertEqual(response["status"], "Success") diff --git a/tests/test_func.py b/tests/test_func.py index d964fd7..373d491 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -1740,7 +1740,7 @@ async def test_initialize_public_method(self): # Verify initialization succeeded self.assertIsNotNone(fq._r) - self.assertIsNotNone(fq._lua_enqueue) + self.assertIsNotNone(fq._scripts.enqueue) # Cleanup await fq.close() @@ -1831,7 +1831,10 @@ def mock_redis_constructor(**kwargs): return mock_redis_instance # Patch Redis to intercept the initialization - with unittest.mock.patch("fq.queue.Redis", side_effect=mock_redis_constructor): + with unittest.mock.patch( + "fq.redis.AsyncRedis", + side_effect=mock_redis_constructor, + ): fq = FQ(config) await fq.initialize() diff --git a/tests/test_sync_queue.py b/tests/test_sync_queue.py index 7616836..5d3a58c 100644 --- a/tests/test_sync_queue.py +++ b/tests/test_sync_queue.py @@ -39,10 +39,10 @@ def test_import_namespace(self): def test_initialize_close_and_reload_scripts(self): self.assertIs(self.queue.redis_client(), self.queue._r) - self.assertIsNotNone(self.queue._lua_enqueue) + self.assertIsNotNone(self.queue._scripts.enqueue) self.queue.reload_lua_scripts() - self.assertIsNotNone(self.queue._lua_enqueue) + self.assertIsNotNone(self.queue._scripts.enqueue) self.assertTrue(self.queue.deep_status()) self.queue.close() @@ -308,6 +308,16 @@ async def scenario(): ) sync_job = await async_queue.dequeue(queue_type=self.queue_type) self.assertEqual(sync_job["status"], "success") + self.assertEqual( + set(sync_job), + { + "status", + "queue_id", + "job_id", + "payload", + "requeues_remaining", + }, + ) self.assertEqual(sync_job["job_id"], sync_job_id) self.assertEqual(sync_job["payload"], {"source": "sync"}) self.assertEqual( @@ -330,6 +340,16 @@ async def scenario(): await asyncio.sleep(0.01) async_job = sync_queue.dequeue(queue_type=self.queue_type) self.assertEqual(async_job["status"], "success") + self.assertEqual( + set(async_job), + { + "status", + "queue_id", + "job_id", + "payload", + "requeues_remaining", + }, + ) self.assertEqual(async_job["job_id"], async_job_id) self.assertEqual(async_job["payload"], {"source": "async"}) self.assertEqual( From 7ab3975c9d29bd16dfa4b5757d3a2e50243594ed Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 20:08:46 +0100 Subject: [PATCH 07/12] Refactors queue logic to centralize shared behavior Extracts shared queue logic into a new base class to reduce code duplication between async and sync clients. Improves maintainability by consolidating argument validation, Redis command construction, and response formatting in one place. Updates usage to subclass the new base and simplifies methods accordingly. --- src/fq/base.py | 224 +++++++++++++++++++++++++++++++++++++++++++ src/fq/lua.py | 2 +- src/fq/queue.py | 205 ++++++++++----------------------------- src/fq/sync/queue.py | 199 +++++++++----------------------------- 4 files changed, 317 insertions(+), 313 deletions(-) create mode 100644 src/fq/base.py diff --git a/src/fq/base.py b/src/fq/base.py new file mode 100644 index 0000000..190af64 --- /dev/null +++ b/src/fq/base.py @@ -0,0 +1,224 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. + +from dataclasses import dataclass + +from fq.config import FQConfig +from fq.exceptions import BadArgumentException +from fq.keys import RedisKeys +from fq.responses import ( + decode_redis_value, + format_dequeue_response, + format_metrics_counts, + format_queue_ids, + format_queue_types, +) +from fq.utils import generate_epoch +from fq.validators import ( + validate_clear_queue_arguments, + validate_dequeue_arguments, + validate_enqueue_arguments, + validate_finish_arguments, + validate_get_queue_length_arguments, + validate_interval_arguments, + validate_metrics_arguments, +) + + +@dataclass(frozen=True) +class ClearQueuePlan: + primary_set: str + job_queue: str + payload_hash: str + interval_hash: str + interval_member: str + queue_type: str + queue_id: str + + def payload_member(self, job_id): + return "%s:%s:%s" % (self.queue_type, self.queue_id, job_id) + + +class BaseFQ(object): + """Shared non-I/O behavior for async and sync FQ clients.""" + + def __init__(self, config): + self._r = None + self._scripts = None + self.config = FQConfig.from_mapping(config) + self._keys = RedisKeys(self.config.redis.key_prefix) + + self._key_prefix = self.config.redis.key_prefix + self._job_expire_interval = int(self.config.job_expire_interval) + self._default_job_requeue_limit = int( + self.config.default_job_requeue_limit + ) + + def redis_client(self): + return self._r + + def _current_timestamp(self): + return str(generate_epoch()) + + def _build_enqueue_call( + self, + payload, + interval, + job_id, + queue_id, + queue_type, + requeue_limit, + ): + enqueue_args = validate_enqueue_arguments( + payload, + interval, + job_id, + queue_id, + queue_type, + requeue_limit, + self._default_job_requeue_limit, + ) + keys = [self._key_prefix, queue_type] + args = [ + self._current_timestamp(), + queue_id, + job_id, + enqueue_args.serialized_payload, + interval, + enqueue_args.requeue_limit, + ] + return keys, args + + def _build_dequeue_call(self, queue_type): + validate_dequeue_arguments(queue_type) + return [self._key_prefix, queue_type], [ + self._current_timestamp(), + self._job_expire_interval, + ] + + def _build_finish_call(self, job_id, queue_id, queue_type): + validate_finish_arguments(job_id, queue_id, queue_type) + return [self._key_prefix, queue_type], [queue_id, job_id] + + def _build_interval_call(self, interval, queue_id, queue_type): + validate_interval_arguments(interval, queue_id, queue_type) + keys = [ + self._keys.interval_hash, + self._keys.interval_member(queue_type, queue_id), + ] + return keys, [interval] + + def _build_requeue_call(self, queue_type, timestamp): + queue_type = decode_redis_value(queue_type) + return [self._key_prefix, queue_type], [timestamp] + + def _build_global_metrics_call(self): + return [self._key_prefix], [self._current_timestamp()] + + def _build_queue_metrics_call(self, queue_type, queue_id): + return [self._keys.job_queue(queue_type, queue_id)], [self._current_timestamp()] + + def _validate_metrics_call(self, queue_type, queue_id): + validate_metrics_arguments(queue_type, queue_id) + if not queue_type and queue_id: + raise BadArgumentException( + "`queue_id` should be accompanied by `queue_type`." + ) + + def _queue_type_metrics_keys(self, queue_type): + return ( + self._keys.ready_queue_set(queue_type), + self._keys.active_queue_set(queue_type), + ) + + def _queue_length_key(self, queue_type, queue_id): + validate_get_queue_length_arguments(queue_type, queue_id) + return self._keys.job_queue(queue_type, queue_id) + + def _clear_queue_plan(self, queue_type, queue_id): + validate_clear_queue_arguments(queue_type, queue_id) + return ClearQueuePlan( + primary_set=self._keys.ready_queue_set(queue_type), + job_queue=self._keys.job_queue(queue_type, queue_id), + payload_hash=self._keys.payload_hash, + interval_hash=self._keys.interval_hash, + interval_member=self._keys.interval_member(queue_type, queue_id), + queue_type=queue_type, + queue_id=queue_id, + ) + + def _finish_response(self, finish_response): + if finish_response == 0: + return {"status": "failure"} + return {"status": "success"} + + def _interval_response(self, interval_response): + if interval_response == 0: + return {"status": "failure"} + return {"status": "success"} + + def _dequeue_response(self, dequeue_response): + return format_dequeue_response(dequeue_response) + + def _global_metrics_response( + self, + active_queue_types, + ready_queue_types, + enqueue_details, + dequeue_details, + ): + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, + dequeue_details, + ) + return { + "status": "success", + "queue_types": format_queue_types(active_queue_types, ready_queue_types), + "enqueue_counts": enqueue_counts, + "dequeue_counts": dequeue_counts, + } + + def _queue_type_metrics_response(self, ready_queues, active_queues): + return { + "status": "success", + "queue_ids": format_queue_ids(ready_queues, active_queues), + } + + def _queue_metrics_response( + self, + queue_length, + enqueue_details, + dequeue_details, + ): + enqueue_counts, dequeue_counts = format_metrics_counts( + enqueue_details, + dequeue_details, + ) + return { + "status": "success", + "queue_length": int(queue_length), + "enqueue_counts": enqueue_counts, + "dequeue_counts": dequeue_counts, + } + + def _decode_redis_value(self, value): + return decode_redis_value(value) + + def _decode_requeue_job(self, job): + queue_id, job_id = decode_redis_value(job).split(":") + return queue_id, job_id + + def _clear_queue_empty_response(self): + return {"status": "Failure", "message": "No queued calls found"} + + def _clear_queue_removed_response(self): + return { + "status": "Success", + "message": "Successfully removed all queued calls", + } + + def _clear_queue_purged_response(self): + return { + "status": "Success", + "message": "Successfully removed all queued calls and purged related resources", + } diff --git a/src/fq/lua.py b/src/fq/lua.py index c05442b..12387c0 100644 --- a/src/fq/lua.py +++ b/src/fq/lua.py @@ -9,7 +9,7 @@ @dataclass(frozen=True) -class Lua: +class LuaScripts: enqueue: object dequeue: object finish: object diff --git a/src/fq/queue.py b/src/fq/queue.py index 147ae92..41832c0 100644 --- a/src/fq/queue.py +++ b/src/fq/queue.py @@ -4,44 +4,13 @@ import asyncio -from fq.config import FQConfig -from fq.exceptions import BadArgumentException -from fq.keys import RedisKeys -from fq.lua import Lua +from fq.base import BaseFQ +from fq.lua import LuaScripts from fq.redis import create_async_redis_client, validate_async_redis_connection -from fq.responses import ( - decode_redis_value, - format_dequeue_response, - format_metrics_counts, - format_queue_ids, - format_queue_types, -) -from fq.utils import generate_epoch -from fq.validators import ( - validate_clear_queue_arguments, - validate_dequeue_arguments, - validate_enqueue_arguments, - validate_finish_arguments, - validate_get_queue_length_arguments, - validate_interval_arguments, - validate_metrics_arguments, -) - - -class FQ(object): - """Async Flowdacity Queue API.""" - def __init__(self, config): - self._r = None - self._scripts = None - self.config = FQConfig.from_mapping(config) - self._keys = RedisKeys(self.config.redis.key_prefix) - - self._key_prefix = self.config.redis.key_prefix - self._job_expire_interval = int(self.config.job_expire_interval) - self._default_job_requeue_limit = int( - self.config.default_job_requeue_limit - ) + +class FQ(BaseFQ): + """Async Flowdacity Queue API.""" async def initialize(self): """Set up the async Redis client and register Lua scripts.""" @@ -49,11 +18,8 @@ async def initialize(self): await validate_async_redis_connection(self._r) self._register_lua_scripts() - def redis_client(self): - return self._r - def _register_lua_scripts(self): - self._scripts = Lua.register(self._r) + self._scripts = LuaScripts.register(self._r) def reload_lua_scripts(self): """Lets user reload the Lua scripts at run time.""" @@ -69,78 +35,45 @@ async def enqueue( requeue_limit=None, ): """Enqueue a job into the specified queue_id and queue_type.""" - enqueue_args = validate_enqueue_arguments( + keys, args = self._build_enqueue_call( payload, interval, job_id, queue_id, queue_type, requeue_limit, - self._default_job_requeue_limit, ) - - keys = [self._key_prefix, queue_type] - args = [ - str(generate_epoch()), - queue_id, - job_id, - enqueue_args.serialized_payload, - interval, - enqueue_args.requeue_limit, - ] await self._scripts.enqueue(keys=keys, args=args) return {"status": "queued"} async def dequeue(self, queue_type="default"): """Dequeue a ready job for queue_type, or return failure.""" - validate_dequeue_arguments(queue_type) - - keys = [self._key_prefix, queue_type] - args = [str(generate_epoch()), self._job_expire_interval] - + keys, args = self._build_dequeue_call(queue_type) dequeue_response = await self._scripts.dequeue(keys=keys, args=args) - return format_dequeue_response(dequeue_response) + return self._dequeue_response(dequeue_response) async def finish(self, job_id, queue_id, queue_type="default"): """Mark a dequeued job as completed successfully.""" - validate_finish_arguments(job_id, queue_id, queue_type) - - keys = [self._key_prefix, queue_type] - args = [queue_id, job_id] - + keys, args = self._build_finish_call(job_id, queue_id, queue_type) finish_response = await self._scripts.finish(keys=keys, args=args) - if finish_response == 0: - return {"status": "failure"} - - return {"status": "success"} + return self._finish_response(finish_response) async def interval(self, interval, queue_id, queue_type="default"): """Update the interval for a queue_id and queue_type.""" - validate_interval_arguments(interval, queue_id, queue_type) - - keys = [ - self._keys.interval_hash, - self._keys.interval_member(queue_type, queue_id), - ] - args = [interval] - + keys, args = self._build_interval_call(interval, queue_id, queue_type) interval_response = await self._scripts.interval(keys=keys, args=args) - if interval_response == 0: - return {"status": "failure"} - - return {"status": "success"} + return self._interval_response(interval_response) async def requeue(self): """Re-queue expired active jobs back into their ready queues.""" - timestamp = str(generate_epoch()) + timestamp = self._current_timestamp() active_queue_type_list = await self._r.smembers(self._keys.active_queue_types) for queue_type in active_queue_type_list: - queue_type = decode_redis_value(queue_type) - keys = [self._key_prefix, queue_type] - args = [timestamp] + queue_type = self._decode_redis_value(queue_type) + keys, args = self._build_requeue_call(queue_type, timestamp) job_discard_list = await self._scripts.requeue(keys=keys, args=args) for job in job_discard_list: - queue_id, job_id = decode_redis_value(job).split(":") + queue_id, job_id = self._decode_requeue_job(job) await self.finish( job_id=job_id, queue_id=queue_id, @@ -149,73 +82,50 @@ async def requeue(self): async def metrics(self, queue_type=None, queue_id=None): """Return global, queue-type, or queue-specific metrics.""" - validate_metrics_arguments(queue_type, queue_id) + self._validate_metrics_call(queue_type, queue_id) - response = {"status": "failure"} if not queue_type and not queue_id: active_queue_types = await self._r.smembers(self._keys.active_queue_types) ready_queue_types = await self._r.smembers(self._keys.ready_queue_types) - queue_types = format_queue_types(active_queue_types, ready_queue_types) - keys = [self._key_prefix] - args = [str(generate_epoch())] + keys, args = self._build_global_metrics_call() enqueue_details, dequeue_details = await self._scripts.metrics( keys=keys, args=args, ) - enqueue_counts, dequeue_counts = format_metrics_counts( + return self._global_metrics_response( + active_queue_types, + ready_queue_types, enqueue_details, dequeue_details, ) - response.update( - { - "status": "success", - "queue_types": queue_types, - "enqueue_counts": enqueue_counts, - "dequeue_counts": dequeue_counts, - } - ) - return response - if queue_type and not queue_id: + ready_queue_key, active_queue_key = self._queue_type_metrics_keys( + queue_type + ) pipe = self._r.pipeline() - pipe.zrange(self._keys.ready_queue_set(queue_type), 0, -1) - pipe.zrange(self._keys.active_queue_set(queue_type), 0, -1) + pipe.zrange(ready_queue_key, 0, -1) + pipe.zrange(active_queue_key, 0, -1) ready_queues, active_queues = await pipe.execute() - queue_list = format_queue_ids(ready_queues, active_queues) - response.update({"status": "success", "queue_ids": queue_list}) - return response + return self._queue_type_metrics_response(ready_queues, active_queues) if queue_type and queue_id: - keys = [self._keys.job_queue(queue_type, queue_id)] - args = [str(generate_epoch())] + keys, args = self._build_queue_metrics_call(queue_type, queue_id) enqueue_details, dequeue_details = await self._scripts.metrics( keys=keys, args=args, ) - enqueue_counts, dequeue_counts = format_metrics_counts( + queue_length = await self._r.llen( + self._queue_length_key(queue_type, queue_id) + ) + return self._queue_metrics_response( + queue_length, enqueue_details, dequeue_details, ) - queue_length = await self._r.llen(self._keys.job_queue(queue_type, queue_id)) - - response.update( - { - "status": "success", - "queue_length": int(queue_length), - "enqueue_counts": enqueue_counts, - "dequeue_counts": dequeue_counts, - } - ) - return response - if not queue_type and queue_id: - raise BadArgumentException( - "`queue_id` should be accompanied by `queue_type`." - ) - - return response + return {"status": "failure"} async def deep_status(self): """ @@ -226,55 +136,36 @@ async def deep_status(self): async def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): """Clear entries in a queue and optionally purge related resources.""" - validate_clear_queue_arguments(queue_type, queue_id) + plan = self._clear_queue_plan(queue_type, queue_id) - response = {"status": "Failure", "message": "No queued calls found"} - primary_set = self._keys.ready_queue_set(queue_type) - queued_status = await self._r.zrem(primary_set, queue_id) + response = self._clear_queue_empty_response() + queued_status = await self._r.zrem(plan.primary_set, queue_id) if queued_status: - response.update( - { - "status": "Success", - "message": "Successfully removed all queued calls", - } - ) + response = self._clear_queue_removed_response() - job_queue_list = self._keys.job_queue(queue_type, queue_id) if queued_status and purge_all: - job_list = await self._r.lrange(job_queue_list, 0, -1) + job_list = await self._r.lrange(plan.job_queue, 0, -1) pipe = self._r.pipeline() for job_uuid in job_list: if job_uuid is None: continue - job_uuid_str = decode_redis_value(job_uuid) - pipe.hdel( - self._keys.payload_hash, - self._keys.payload_member(queue_type, queue_id, job_uuid_str), - ) + job_uuid = self._decode_redis_value(job_uuid) + pipe.hdel(plan.payload_hash, plan.payload_member(job_uuid)) - pipe.hdel( - self._keys.interval_hash, - self._keys.interval_member(queue_type, queue_id), - ) - pipe.delete(job_queue_list) + pipe.hdel(plan.interval_hash, plan.interval_member) + pipe.delete(plan.job_queue) await pipe.execute() - response.update( - { - "status": "Success", - "message": "Successfully removed all queued calls and purged related resources", - } - ) - else: - await self._r.delete(job_queue_list) + return self._clear_queue_purged_response() + await self._r.delete(plan.job_queue) return response async def get_queue_length(self, queue_type, queue_id): """ Return the current Redis list length for key_prefix:queue_type:queue_id. """ - validate_get_queue_length_arguments(queue_type, queue_id) - return await self._r.llen(self._keys.job_queue(queue_type, queue_id)) + redis_key = self._queue_length_key(queue_type, queue_id) + return await self._r.llen(redis_key) async def close(self): """Cleanly close the underlying Redis client or connection pool.""" diff --git a/src/fq/sync/queue.py b/src/fq/sync/queue.py index e3a749e..e9652ea 100644 --- a/src/fq/sync/queue.py +++ b/src/fq/sync/queue.py @@ -1,56 +1,22 @@ # -*- coding: utf-8 -*- # Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. -from fq.config import FQConfig -from fq.exceptions import BadArgumentException -from fq.keys import RedisKeys -from fq.lua import Lua +from fq.base import BaseFQ +from fq.lua import LuaScripts from fq.redis import create_sync_redis_client, validate_sync_redis_connection -from fq.responses import ( - decode_redis_value, - format_dequeue_response, - format_metrics_counts, - format_queue_ids, - format_queue_types, -) -from fq.utils import generate_epoch -from fq.validators import ( - validate_clear_queue_arguments, - validate_dequeue_arguments, - validate_enqueue_arguments, - validate_finish_arguments, - validate_get_queue_length_arguments, - validate_interval_arguments, - validate_metrics_arguments, -) -class FQ(object): +class FQ(BaseFQ): """Synchronous Flowdacity Queue API.""" - def __init__(self, config): - self._r = None - self._scripts = None - self.config = FQConfig.from_mapping(config) - self._keys = RedisKeys(self.config.redis.key_prefix) - - self._key_prefix = self.config.redis.key_prefix - self._job_expire_interval = int(self.config.job_expire_interval) - self._default_job_requeue_limit = int( - self.config.default_job_requeue_limit - ) - def initialize(self): """Set up the synchronous Redis client and register Lua scripts.""" self._r = create_sync_redis_client(self.config.redis) validate_sync_redis_connection(self._r) self._register_lua_scripts() - def redis_client(self): - return self._r - def _register_lua_scripts(self): - self._scripts = Lua.register(self._r) + self._scripts = LuaScripts.register(self._r) def reload_lua_scripts(self): """Lets user reload the Lua scripts at run time.""" @@ -66,149 +32,91 @@ def enqueue( requeue_limit=None, ): """Enqueue a job into the specified queue_id and queue_type.""" - enqueue_args = validate_enqueue_arguments( + keys, args = self._build_enqueue_call( payload, interval, job_id, queue_id, queue_type, requeue_limit, - self._default_job_requeue_limit, ) - - keys = [self._key_prefix, queue_type] - args = [ - str(generate_epoch()), - queue_id, - job_id, - enqueue_args.serialized_payload, - interval, - enqueue_args.requeue_limit, - ] self._scripts.enqueue(keys=keys, args=args) return {"status": "queued"} def dequeue(self, queue_type="default"): """Dequeue a ready job for queue_type, or return failure.""" - validate_dequeue_arguments(queue_type) - - keys = [self._key_prefix, queue_type] - args = [str(generate_epoch()), self._job_expire_interval] - + keys, args = self._build_dequeue_call(queue_type) dequeue_response = self._scripts.dequeue(keys=keys, args=args) - return format_dequeue_response(dequeue_response) + return self._dequeue_response(dequeue_response) def finish(self, job_id, queue_id, queue_type="default"): """Mark a dequeued job as completed successfully.""" - validate_finish_arguments(job_id, queue_id, queue_type) - - keys = [self._key_prefix, queue_type] - args = [queue_id, job_id] - + keys, args = self._build_finish_call(job_id, queue_id, queue_type) finish_response = self._scripts.finish(keys=keys, args=args) - if finish_response == 0: - return {"status": "failure"} - - return {"status": "success"} + return self._finish_response(finish_response) def interval(self, interval, queue_id, queue_type="default"): """Update the interval for a queue_id and queue_type.""" - validate_interval_arguments(interval, queue_id, queue_type) - - keys = [ - self._keys.interval_hash, - self._keys.interval_member(queue_type, queue_id), - ] - args = [interval] - + keys, args = self._build_interval_call(interval, queue_id, queue_type) interval_response = self._scripts.interval(keys=keys, args=args) - if interval_response == 0: - return {"status": "failure"} - - return {"status": "success"} + return self._interval_response(interval_response) def requeue(self): """Re-queue expired active jobs back into their ready queues.""" - timestamp = str(generate_epoch()) + timestamp = self._current_timestamp() active_queue_type_list = self._r.smembers(self._keys.active_queue_types) for queue_type in active_queue_type_list: - queue_type = decode_redis_value(queue_type) - keys = [self._key_prefix, queue_type] - args = [timestamp] + queue_type = self._decode_redis_value(queue_type) + keys, args = self._build_requeue_call(queue_type, timestamp) job_discard_list = self._scripts.requeue(keys=keys, args=args) for job in job_discard_list: - queue_id, job_id = decode_redis_value(job).split(":") + queue_id, job_id = self._decode_requeue_job(job) self.finish(job_id=job_id, queue_id=queue_id, queue_type=queue_type) def metrics(self, queue_type=None, queue_id=None): """Return global, queue-type, or queue-specific metrics.""" - validate_metrics_arguments(queue_type, queue_id) + self._validate_metrics_call(queue_type, queue_id) - response = {"status": "failure"} if not queue_type and not queue_id: active_queue_types = self._r.smembers(self._keys.active_queue_types) ready_queue_types = self._r.smembers(self._keys.ready_queue_types) - queue_types = format_queue_types(active_queue_types, ready_queue_types) - keys = [self._key_prefix] - args = [str(generate_epoch())] + keys, args = self._build_global_metrics_call() enqueue_details, dequeue_details = self._scripts.metrics( keys=keys, args=args, ) - enqueue_counts, dequeue_counts = format_metrics_counts( + return self._global_metrics_response( + active_queue_types, + ready_queue_types, enqueue_details, dequeue_details, ) - response.update( - { - "status": "success", - "queue_types": queue_types, - "enqueue_counts": enqueue_counts, - "dequeue_counts": dequeue_counts, - } - ) - return response - if queue_type and not queue_id: + ready_queue_key, active_queue_key = self._queue_type_metrics_keys( + queue_type + ) pipe = self._r.pipeline() - pipe.zrange(self._keys.ready_queue_set(queue_type), 0, -1) - pipe.zrange(self._keys.active_queue_set(queue_type), 0, -1) + pipe.zrange(ready_queue_key, 0, -1) + pipe.zrange(active_queue_key, 0, -1) ready_queues, active_queues = pipe.execute() - queue_list = format_queue_ids(ready_queues, active_queues) - response.update({"status": "success", "queue_ids": queue_list}) - return response + return self._queue_type_metrics_response(ready_queues, active_queues) if queue_type and queue_id: - keys = [self._keys.job_queue(queue_type, queue_id)] - args = [str(generate_epoch())] + keys, args = self._build_queue_metrics_call(queue_type, queue_id) enqueue_details, dequeue_details = self._scripts.metrics( keys=keys, args=args, ) - enqueue_counts, dequeue_counts = format_metrics_counts( + queue_length = self._r.llen(self._queue_length_key(queue_type, queue_id)) + return self._queue_metrics_response( + queue_length, enqueue_details, dequeue_details, ) - queue_length = self._r.llen(self._keys.job_queue(queue_type, queue_id)) - - response.update( - { - "status": "success", - "queue_length": int(queue_length), - "enqueue_counts": enqueue_counts, - "dequeue_counts": dequeue_counts, - } - ) - return response - - if not queue_type and queue_id: - raise BadArgumentException( - "`queue_id` should be accompanied by `queue_type`." - ) - return response + return {"status": "failure"} def deep_status(self): """ @@ -219,55 +127,36 @@ def deep_status(self): def clear_queue(self, queue_type=None, queue_id=None, purge_all=False): """Clear entries in a queue and optionally purge related resources.""" - validate_clear_queue_arguments(queue_type, queue_id) + plan = self._clear_queue_plan(queue_type, queue_id) - response = {"status": "Failure", "message": "No queued calls found"} - primary_set = self._keys.ready_queue_set(queue_type) - queued_status = self._r.zrem(primary_set, queue_id) + response = self._clear_queue_empty_response() + queued_status = self._r.zrem(plan.primary_set, queue_id) if queued_status: - response.update( - { - "status": "Success", - "message": "Successfully removed all queued calls", - } - ) + response = self._clear_queue_removed_response() - job_queue_list = self._keys.job_queue(queue_type, queue_id) if queued_status and purge_all: - job_list = self._r.lrange(job_queue_list, 0, -1) + job_list = self._r.lrange(plan.job_queue, 0, -1) pipe = self._r.pipeline() for job_uuid in job_list: if job_uuid is None: continue - job_uuid_str = decode_redis_value(job_uuid) - pipe.hdel( - self._keys.payload_hash, - self._keys.payload_member(queue_type, queue_id, job_uuid_str), - ) + job_uuid = self._decode_redis_value(job_uuid) + pipe.hdel(plan.payload_hash, plan.payload_member(job_uuid)) - pipe.hdel( - self._keys.interval_hash, - self._keys.interval_member(queue_type, queue_id), - ) - pipe.delete(job_queue_list) + pipe.hdel(plan.interval_hash, plan.interval_member) + pipe.delete(plan.job_queue) pipe.execute() - response.update( - { - "status": "Success", - "message": "Successfully removed all queued calls and purged related resources", - } - ) - else: - self._r.delete(job_queue_list) + return self._clear_queue_purged_response() + self._r.delete(plan.job_queue) return response def get_queue_length(self, queue_type, queue_id): """ Return the current Redis list length for key_prefix:queue_type:queue_id. """ - validate_get_queue_length_arguments(queue_type, queue_id) - return self._r.llen(self._keys.job_queue(queue_type, queue_id)) + redis_key = self._queue_length_key(queue_type, queue_id) + return self._r.llen(redis_key) def close(self): """Close the underlying synchronous Redis client.""" From 6ea73283e92a8bebe40390701e05dc25c55dd6e8 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 20:41:48 +0100 Subject: [PATCH 08/12] Adds Redis password support and improves queue ID formatting Passes the configured password to all Redis client initializations, including cluster and UNIX socket connections, enhancing security and compatibility with protected instances. Refines queue ID formatting logic to ensure correct deduplication of ready and active queues. Extends unit tests to cover password handling and queue ID formatting. --- src/fq/redis.py | 4 +++ src/fq/responses.py | 8 +++-- tests/test_edge_cases.py | 72 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 6 deletions(-) diff --git a/src/fq/redis.py b/src/fq/redis.py index 9eb1f28..916ccdf 100644 --- a/src/fq/redis.py +++ b/src/fq/redis.py @@ -14,6 +14,7 @@ def create_async_redis_client(redis_config): return AsyncRedis( db=redis_config.db, unix_socket_path=redis_config.unix_socket_path, + password=redis_config.password, ) if redis_config.conn_type == "tcp_sock": @@ -27,6 +28,7 @@ def create_async_redis_client(redis_config): return AsyncRedisCluster( startup_nodes=startup_nodes, decode_responses=False, + password=redis_config.password, socket_timeout=5, ) @@ -45,6 +47,7 @@ def create_sync_redis_client(redis_config): return SyncRedis( db=redis_config.db, unix_socket_path=redis_config.unix_socket_path, + password=redis_config.password, ) if redis_config.conn_type == "tcp_sock": @@ -53,6 +56,7 @@ def create_sync_redis_client(redis_config): host=redis_config.host, port=int(redis_config.port), decode_responses=False, + password=redis_config.password, socket_timeout=5, ) diff --git a/src/fq/responses.py b/src/fq/responses.py index 04092ed..6eb7091 100644 --- a/src/fq/responses.py +++ b/src/fq/responses.py @@ -46,6 +46,8 @@ def format_queue_types(active_queue_types, ready_queue_types): def format_queue_ids(ready_queues, active_queues): - active_queues = [decode_redis_value(i).split(":")[0] for i in active_queues] - all_queue_set = set(ready_queues) | set(active_queues) - return convert_to_str(all_queue_set) + ready_queue_ids = {decode_redis_value(queue) for queue in ready_queues} + active_queue_ids = { + decode_redis_value(queue).split(":")[0] for queue in active_queues + } + return convert_to_str(ready_queue_ids | active_queue_ids) diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index a48d38f..9da054b 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -8,15 +8,25 @@ from unittest.mock import patch from fq import FQ -from fq.utils import is_valid_identifier +from fq.config import FQConfig from fq.exceptions import BadArgumentException, FQException +from fq.redis import create_async_redis_client, create_sync_redis_client +from fq.responses import format_queue_ids +from fq.utils import is_valid_identifier from tests.config import build_test_config class FakeCluster: - def __init__(self, startup_nodes=None, decode_responses=False, socket_timeout=None): + def __init__( + self, + startup_nodes=None, + decode_responses=False, + password=None, + socket_timeout=None, + ): self.startup_nodes = startup_nodes or [] self.decode_responses = decode_responses + self.password = password self.socket_timeout = socket_timeout def register_script(self, _): @@ -113,6 +123,12 @@ async def delete(self, key): self.deleted_keys.append(key) +class RecordingRedisClient: + def __init__(self, *args, **kwargs): + self.args = args + self.kwargs = kwargs + + class TestEdgeCases(unittest.IsolatedAsyncioTestCase): # qlty-ignore(radarlint-python:python:S5899): unittest lifecycle hook. async def asyncSetUp(self): @@ -143,12 +159,17 @@ async def test_initialize_fails_fast_on_bad_redis(self): async def test_cluster_initialization(self): """Covers clustered Redis path (queue.py lines 69-75, 104-106).""" config = build_test_config( - redis={"key_prefix": "test_fq_cluster", "clustered": True} + redis={ + "key_prefix": "test_fq_cluster", + "clustered": True, + "password": "cluster-password", + } ) with patch("fq.redis.AsyncRedisCluster", FakeCluster): fq = FQ(config) await fq.initialize() self.assertIsInstance(fq.redis_client(), FakeCluster) + self.assertEqual(fq.redis_client().password, "cluster-password") await fq.close() def test_clustered_config_must_be_boolean(self): @@ -218,6 +239,51 @@ def test_is_valid_identifier_non_string(self): self.assertFalse(is_valid_identifier(None)) self.assertFalse(is_valid_identifier(["a"])) + def test_format_queue_ids_deduplicates_ready_and_active_queues(self): + queue_ids = format_queue_ids( + ready_queues=[b"johndoe", b"ready-only"], + active_queues=[b"johndoe:job-1", "active-only:job-2"], + ) + + self.assertEqual(set(queue_ids), {"johndoe", "ready-only", "active-only"}) + self.assertEqual(len(queue_ids), 3) + + def test_redis_factories_pass_password_to_unix_socket_clients(self): + config = FQConfig.from_mapping( + build_test_config( + redis={ + "conn_type": "unix_sock", + "password": "socket-password", + } + ) + ) + + with patch("fq.redis.AsyncRedis", RecordingRedisClient): + async_client = create_async_redis_client(config.redis) + with patch("fq.redis.SyncRedis", RecordingRedisClient): + sync_client = create_sync_redis_client(config.redis) + + self.assertEqual(async_client.kwargs["password"], "socket-password") + self.assertEqual(sync_client.kwargs["password"], "socket-password") + + def test_redis_factories_pass_password_to_cluster_clients(self): + config = FQConfig.from_mapping( + build_test_config( + redis={ + "clustered": True, + "password": "cluster-password", + } + ) + ) + + with patch("fq.redis.AsyncRedisCluster", RecordingRedisClient): + async_client = create_async_redis_client(config.redis) + with patch("fq.redis.SyncRedisCluster", RecordingRedisClient): + sync_client = create_sync_redis_client(config.redis) + + self.assertEqual(async_client.kwargs["password"], "cluster-password") + self.assertEqual(sync_client.kwargs["password"], "cluster-password") + async def test_clear_queue_purge_all_with_mixed_job_ids(self): """Covers purge_all loop branches (queue.py lines 463-468, 474-479).""" fq = FQ(self.config) From 8ac72390b3c91bc999a4ddd158661a7a91a1dc67 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 20:57:49 +0100 Subject: [PATCH 09/12] Enforces Redis config validation and async sync compatibility Validates that the 'clustered' Redis config flag is a boolean and that the port falls within the valid range, raising clear exceptions for misconfiguration. Refactors Lua script registration for maintainability. Updates async and sync interop in tests to use threads, improving compatibility and test reliability. Enhances error coverage for edge cases. --- src/fq/config.py | 13 +++++++++++-- src/fq/lua.py | 24 ++++++++++++------------ src/fq/redis.py | 6 ++---- src/fq/utils.py | 4 ++-- tests/test_edge_cases.py | 25 +++++++++++++++++++++++++ tests/test_sync_queue.py | 18 ++++++++++++------ 6 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/fq/config.py b/src/fq/config.py index 6a96dd4..435070f 100644 --- a/src/fq/config.py +++ b/src/fq/config.py @@ -130,6 +130,8 @@ def _validate_fq_config(fq_config): def _validate_connection_config(redis_config): + _validate_clustered_config(redis_config) + if redis_config["conn_type"] == "unix_sock": _validate_unix_socket_config(redis_config) return @@ -137,6 +139,11 @@ def _validate_connection_config(redis_config): _validate_tcp_socket_config(redis_config) +def _validate_clustered_config(redis_config): + if "clustered" in redis_config and not isinstance(redis_config["clustered"], bool): + raise FQException("Invalid config: redis.clustered must be a boolean") + + def _validate_unix_socket_config(redis_config): unix_socket_path = _require_config_value(redis_config, "redis", "unix_socket_path") if not _is_non_empty_string(unix_socket_path): @@ -154,8 +161,10 @@ def _validate_tcp_socket_config(redis_config): if not _is_int_not_bool(port): raise FQException("Invalid config: redis.port must be an integer") - if "clustered" in redis_config and not isinstance(redis_config["clustered"], bool): - raise FQException("Invalid config: redis.clustered must be a boolean") + if port < 1 or port > 65535: + raise FQException( + "Invalid config: redis.port must be an integer between 1 and 65535" + ) def _validate_optional_redis_config(redis_config): diff --git a/src/fq/lua.py b/src/fq/lua.py index 12387c0..cd9affa 100644 --- a/src/fq/lua.py +++ b/src/fq/lua.py @@ -1,27 +1,27 @@ # -*- coding: utf-8 -*- # Copyright (c) 2025 Flowdacity Development Team. See LICENSE.txt for details. -from dataclasses import dataclass +from dataclasses import dataclass, fields from pathlib import Path - - -LUA_SCRIPT_NAMES = ("enqueue", "dequeue", "finish", "interval", "requeue", "metrics") +from typing import Any @dataclass(frozen=True) class LuaScripts: - enqueue: object - dequeue: object - finish: object - interval: object - requeue: object - metrics: object + enqueue: Any + dequeue: Any + finish: Any + interval: Any + requeue: Any + metrics: Any @classmethod def register(cls, redis_client): registered_scripts = { - script_name: redis_client.register_script(cls._read_script(script_name)) - for script_name in LUA_SCRIPT_NAMES + script_field.name: redis_client.register_script( + cls._read_script(script_field.name) + ) + for script_field in fields(cls) } return cls(**registered_scripts) diff --git a/src/fq/redis.py b/src/fq/redis.py index 916ccdf..404bb3c 100644 --- a/src/fq/redis.py +++ b/src/fq/redis.py @@ -4,6 +4,7 @@ from redis import Redis as SyncRedis from redis import RedisCluster as SyncRedisCluster from redis.asyncio import Redis as AsyncRedis +from redis.asyncio.cluster import ClusterNode as AsyncClusterNode from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster from fq.exceptions import FQException @@ -20,10 +21,7 @@ def create_async_redis_client(redis_config): if redis_config.conn_type == "tcp_sock": if redis_config.clustered: startup_nodes = [ - { - "host": redis_config.host, - "port": int(redis_config.port), - } + AsyncClusterNode(redis_config.host, int(redis_config.port)), ] return AsyncRedisCluster( startup_nodes=startup_nodes, diff --git a/src/fq/utils.py b/src/fq/utils.py index 11faad3..29c6168 100644 --- a/src/fq/utils.py +++ b/src/fq/utils.py @@ -74,8 +74,8 @@ def convert_to_str(queue_set): """Takes set and decodes bytes to string""" queue_list = [] for queue in list(queue_set): - try: + if isinstance(queue, (bytes, bytearray)): queue_list.append(queue.decode("utf-8")) - except Exception: + else: queue_list.append(queue) return queue_list diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 9da054b..0095bf1 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -170,6 +170,9 @@ async def test_cluster_initialization(self): await fq.initialize() self.assertIsInstance(fq.redis_client(), FakeCluster) self.assertEqual(fq.redis_client().password, "cluster-password") + startup_node = fq.redis_client().startup_nodes[0] + self.assertEqual(startup_node.host, "127.0.0.1") + self.assertEqual(startup_node.port, 6379) await fq.close() def test_clustered_config_must_be_boolean(self): @@ -179,6 +182,18 @@ def test_clustered_config_must_be_boolean(self): ): FQ(config) + def test_unix_socket_clustered_config_must_be_boolean(self): + config = build_test_config( + redis={ + "conn_type": "unix_sock", + "clustered": "true", + } + ) + with self.assertRaisesRegex( + FQException, "Invalid config: redis.clustered must be a boolean" + ): + FQ(config) + def test_missing_required_config_key_raises_with_path(self): config = build_test_config() del config["redis"]["key_prefix"] @@ -193,6 +208,16 @@ def test_invalid_config_value_raises_with_path(self): ): FQ(config) + def test_invalid_redis_port_range_raises(self): + for port in (0, -1, 65536): + with self.subTest(port=port): + config = build_test_config(redis={"port": port}) + with self.assertRaisesRegex( + FQException, + "Invalid config: redis.port must be an integer between 1 and 65535", + ): + FQ(config) + async def test_dequeue_payload_none(self): """Covers dequeue branch where payload is None (queue.py line 212).""" fq = FQ(self.config) diff --git a/tests/test_sync_queue.py b/tests/test_sync_queue.py index 5d3a58c..d3dd284 100644 --- a/tests/test_sync_queue.py +++ b/tests/test_sync_queue.py @@ -294,12 +294,13 @@ async def scenario(): sync_queue = FQ(config) await async_queue.initialize() - sync_queue.initialize() + await asyncio.to_thread(sync_queue.initialize) await async_queue._r.flushdb() try: sync_job_id = self._job_id() - sync_queue.enqueue( + await asyncio.to_thread( + sync_queue.enqueue, payload={"source": "sync"}, interval=1, job_id=sync_job_id, @@ -338,7 +339,10 @@ async def scenario(): queue_type=self.queue_type, ) await asyncio.sleep(0.01) - async_job = sync_queue.dequeue(queue_type=self.queue_type) + async_job = await asyncio.to_thread( + sync_queue.dequeue, + queue_type=self.queue_type, + ) self.assertEqual(async_job["status"], "success") self.assertEqual( set(async_job), @@ -353,7 +357,8 @@ async def scenario(): self.assertEqual(async_job["job_id"], async_job_id) self.assertEqual(async_job["payload"], {"source": "async"}) self.assertEqual( - sync_queue.finish( + await asyncio.to_thread( + sync_queue.finish, queue_type=self.queue_type, queue_id=async_job["queue_id"], job_id=async_job["job_id"], @@ -361,8 +366,9 @@ async def scenario(): {"status": "success"}, ) finally: - sync_queue._r.flushdb() - sync_queue.close() + if sync_queue._r is not None: + await asyncio.to_thread(sync_queue._r.flushdb) + await asyncio.to_thread(sync_queue.close) await async_queue.close() asyncio.run(scenario()) From 4d492d81431e85908aaa7d1be28b40907e961c89 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 21:06:42 +0100 Subject: [PATCH 10/12] Refactors config validation for clarity and separation Splits and reorganizes configuration validation logic to clearly separate Redis and application (FQ) config concerns. Moves validation methods into their respective classes for better encapsulation and maintainability. Simplifies and clarifies value checks, improving code readability and making validation flows more explicit. --- src/fq/config.py | 240 +++++++++++++++++++++++++---------------------- 1 file changed, 127 insertions(+), 113 deletions(-) diff --git a/src/fq/config.py b/src/fq/config.py index 435070f..c3ad6b8 100644 --- a/src/fq/config.py +++ b/src/fq/config.py @@ -22,152 +22,166 @@ class RedisConfig: clustered: bool = False password: str | None = None - -@dataclass(frozen=True) -class FQConfig: - redis: RedisConfig - job_expire_interval: int - job_requeue_interval: int - default_job_requeue_limit: int - @classmethod def from_mapping(cls, config): - normalized = _normalize_config_sections(config) - _require_config_sections(normalized) - - redis_config = normalized["redis"] - fq_config = normalized["fq"] - - _validate_redis_config(redis_config) - _validate_fq_config(fq_config) - _validate_connection_config(redis_config) - _validate_optional_redis_config(redis_config) + cls._validate_required(config) + cls._validate_connection(config) + cls._validate_optional(config) return cls( - redis=RedisConfig( - key_prefix=redis_config["key_prefix"], - conn_type=redis_config["conn_type"], - db=redis_config["db"], - host=redis_config.get("host"), - port=redis_config.get("port"), - unix_socket_path=redis_config.get("unix_socket_path"), - clustered=redis_config.get("clustered", False), - password=redis_config.get("password"), - ), - job_expire_interval=fq_config["job_expire_interval"], - job_requeue_interval=fq_config["job_requeue_interval"], - default_job_requeue_limit=fq_config["default_job_requeue_limit"], + key_prefix=config["key_prefix"], + conn_type=config["conn_type"], + db=config["db"], + host=config.get("host"), + port=config.get("port"), + unix_socket_path=config.get("unix_socket_path"), + clustered=config.get("clustered", False), + password=config.get("password"), ) + @classmethod + def _validate_required(cls, config): + key_prefix = cls._require_value(config, "key_prefix") + if not cls._is_non_empty_string(key_prefix): + raise FQException( + "Invalid config: redis.key_prefix must be a non-empty string" + ) -def _normalize_config_sections(config): - if not isinstance(config, Mapping): - raise FQException("Config must be a mapping with redis and fq sections") + conn_type = cls._require_value(config, "conn_type") + if conn_type not in REDIS_CONN_TYPES: + raise FQException( + "Invalid config: redis.conn_type must be 'tcp_sock' or 'unix_sock'" + ) - normalized = {} - for section_name, section_values in config.items(): - if not isinstance(section_values, Mapping): - raise FQException("Config section '%s' must be a mapping" % section_name) + db = cls._require_value(config, "db") + if not cls._is_int_not_bool(db): + raise FQException("Invalid config: redis.db must be an integer") - normalized[str(section_name)] = { - str(option): value for option, value in section_values.items() - } + @classmethod + def _validate_connection(cls, config): + cls._validate_clustered(config) - return normalized + if config["conn_type"] == "unix_sock": + cls._validate_unix_socket(config) + return + cls._validate_tcp_socket(config) -def _require_config_sections(config): - if "redis" not in config or "fq" not in config: - raise FQException("Config missing required sections: redis, fq") + @classmethod + def _validate_clustered(cls, config): + if "clustered" in config and not isinstance(config["clustered"], bool): + raise FQException("Invalid config: redis.clustered must be a boolean") + @classmethod + def _validate_unix_socket(cls, config): + unix_socket_path = cls._require_value(config, "unix_socket_path") + if not cls._is_non_empty_string(unix_socket_path): + raise FQException( + "Invalid config: redis.unix_socket_path must be a non-empty string" + ) -def _require_config_value(config, section_name, option_name): - if option_name not in config: - raise FQException("Missing config: %s.%s" % (section_name, option_name)) + @classmethod + def _validate_tcp_socket(cls, config): + host = cls._require_value(config, "host") + if not cls._is_non_empty_string(host): + raise FQException("Invalid config: redis.host must be a non-empty string") - return config[option_name] + port = cls._require_value(config, "port") + if not cls._is_int_not_bool(port): + raise FQException("Invalid config: redis.port must be an integer") + if port < 1 or port > 65535: + raise FQException( + "Invalid config: redis.port must be an integer between 1 and 65535" + ) -def _is_non_empty_string(value): - return isinstance(value, str) and bool(value) + @classmethod + def _validate_optional(cls, config): + if "password" in config and config["password"] is not None: + if not isinstance(config["password"], str): + raise FQException("Invalid config: redis.password must be a string") + @staticmethod + def _require_value(config, option_name): + if option_name not in config: + raise FQException("Missing config: redis.%s" % option_name) -def _is_int_not_bool(value): - return isinstance(value, int) and not isinstance(value, bool) + return config[option_name] + @staticmethod + def _is_non_empty_string(value): + return isinstance(value, str) and bool(value) -def _validate_redis_config(redis_config): - key_prefix = _require_config_value(redis_config, "redis", "key_prefix") - if not _is_non_empty_string(key_prefix): - raise FQException("Invalid config: redis.key_prefix must be a non-empty string") + @staticmethod + def _is_int_not_bool(value): + return isinstance(value, int) and not isinstance(value, bool) - conn_type = _require_config_value(redis_config, "redis", "conn_type") - if conn_type not in REDIS_CONN_TYPES: - raise FQException( - "Invalid config: redis.conn_type must be 'tcp_sock' or 'unix_sock'" - ) - db = _require_config_value(redis_config, "redis", "db") - if not _is_int_not_bool(db): - raise FQException("Invalid config: redis.db must be an integer") +@dataclass(frozen=True) +class FQConfig: + redis: RedisConfig + job_expire_interval: int + job_requeue_interval: int + default_job_requeue_limit: int + @classmethod + def from_mapping(cls, config): + normalized = cls._normalize_sections(config) + cls._require_sections(normalized) -def _validate_fq_config(fq_config): - for option_name in ("job_expire_interval", "job_requeue_interval"): - value = _require_config_value(fq_config, "fq", option_name) - if not is_valid_interval(value): - raise FQException( - "Invalid config: fq.%s must be a positive integer" % option_name - ) + fq_config = normalized["fq"] + cls._validate_fq_section(fq_config) - default_requeue_limit = _require_config_value( - fq_config, "fq", "default_job_requeue_limit" - ) - if not is_valid_requeue_limit(default_requeue_limit): - raise FQException( - "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" + return cls( + redis=RedisConfig.from_mapping(normalized["redis"]), + job_expire_interval=fq_config["job_expire_interval"], + job_requeue_interval=fq_config["job_requeue_interval"], + default_job_requeue_limit=fq_config["default_job_requeue_limit"], ) + @staticmethod + def _normalize_sections(config): + if not isinstance(config, Mapping): + raise FQException("Config must be a mapping with redis and fq sections") -def _validate_connection_config(redis_config): - _validate_clustered_config(redis_config) - - if redis_config["conn_type"] == "unix_sock": - _validate_unix_socket_config(redis_config) - return + normalized = {} + for section_name, section_values in config.items(): + if not isinstance(section_values, Mapping): + raise FQException( + "Config section '%s' must be a mapping" % section_name + ) - _validate_tcp_socket_config(redis_config) + normalized[str(section_name)] = { + str(option): value for option, value in section_values.items() + } + return normalized -def _validate_clustered_config(redis_config): - if "clustered" in redis_config and not isinstance(redis_config["clustered"], bool): - raise FQException("Invalid config: redis.clustered must be a boolean") - - -def _validate_unix_socket_config(redis_config): - unix_socket_path = _require_config_value(redis_config, "redis", "unix_socket_path") - if not _is_non_empty_string(unix_socket_path): - raise FQException( - "Invalid config: redis.unix_socket_path must be a non-empty string" - ) + @staticmethod + def _require_sections(config): + if "redis" not in config or "fq" not in config: + raise FQException("Config missing required sections: redis, fq") - -def _validate_tcp_socket_config(redis_config): - host = _require_config_value(redis_config, "redis", "host") - if not _is_non_empty_string(host): - raise FQException("Invalid config: redis.host must be a non-empty string") - - port = _require_config_value(redis_config, "redis", "port") - if not _is_int_not_bool(port): - raise FQException("Invalid config: redis.port must be an integer") - - if port < 1 or port > 65535: - raise FQException( - "Invalid config: redis.port must be an integer between 1 and 65535" + @classmethod + def _validate_fq_section(cls, config): + for option_name in ("job_expire_interval", "job_requeue_interval"): + value = cls._require_value(config, option_name) + if not is_valid_interval(value): + raise FQException( + "Invalid config: fq.%s must be a positive integer" % option_name + ) + + default_requeue_limit = cls._require_value( + config, "default_job_requeue_limit" ) + if not is_valid_requeue_limit(default_requeue_limit): + raise FQException( + "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" + ) + @staticmethod + def _require_value(config, option_name): + if option_name not in config: + raise FQException("Missing config: fq.%s" % option_name) -def _validate_optional_redis_config(redis_config): - if "password" in redis_config and redis_config["password"] is not None: - if not isinstance(redis_config["password"], str): - raise FQException("Invalid config: redis.password must be a string") + return config[option_name] From cb8e8dce15a3619f8bbc18878e564188fbc35ef8 Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 21:37:37 +0100 Subject: [PATCH 11/12] Replaces "fq" config section with "queue" for clarity Standardizes configuration by moving queue-related options from a generic "fq" section to a dedicated "queue" section with explicit validation and structure. Removes the key prefix from the Redis config, centralizing it under queue settings to reduce redundancy and improve separation of concerns. Updates documentation, code, and tests to reflect the new config schema, and adds stricter validation to reject legacy "fq" sections. Aims to improve configuration clarity and future extensibility. --- README.md | 12 ++--- src/fq/base.py | 8 ++-- src/fq/config.py | 100 ++++++++++++++++++++++----------------- tests/config.py | 4 +- tests/test_edge_cases.py | 28 ++++++++--- tests/test_func.py | 2 +- tests/test_sync_queue.py | 12 +++-- 7 files changed, 99 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index f8a4855..32fa728 100644 --- a/README.md +++ b/README.md @@ -37,14 +37,14 @@ pip install -e . FQ accepts a simple config mapping. Intervals are in milliseconds. ```python config = { - "fq": { + "queue": { + "key_prefix": "queue_server", "job_expire_interval": 5000, "job_requeue_interval": 5000, "default_job_requeue_limit": -1, # -1 retries forever, 0 means no retries }, "redis": { "db": 0, - "key_prefix": "queue_server", "conn_type": "tcp_sock", # or "unix_sock" "host": "127.0.0.1", "port": 6379, @@ -77,14 +77,14 @@ from fq import FQ async def main(): config = { - "fq": { + "queue": { + "key_prefix": "queue_server", "job_expire_interval": 5000, "job_requeue_interval": 5000, "default_job_requeue_limit": -1, }, "redis": { "db": 0, - "key_prefix": "queue_server", "conn_type": "tcp_sock", "host": "127.0.0.1", "port": 6379, @@ -131,14 +131,14 @@ from fq.sync import FQ config = { - "fq": { + "queue": { + "key_prefix": "queue_server", "job_expire_interval": 5000, "job_requeue_interval": 5000, "default_job_requeue_limit": -1, }, "redis": { "db": 0, - "key_prefix": "queue_server", "conn_type": "tcp_sock", "host": "127.0.0.1", "port": 6379, diff --git a/src/fq/base.py b/src/fq/base.py index 190af64..ffef37a 100644 --- a/src/fq/base.py +++ b/src/fq/base.py @@ -46,12 +46,12 @@ def __init__(self, config): self._r = None self._scripts = None self.config = FQConfig.from_mapping(config) - self._keys = RedisKeys(self.config.redis.key_prefix) + self._keys = RedisKeys(self.config.queue.key_prefix) - self._key_prefix = self.config.redis.key_prefix - self._job_expire_interval = int(self.config.job_expire_interval) + self._key_prefix = self.config.queue.key_prefix + self._job_expire_interval = int(self.config.queue.job_expire_interval) self._default_job_requeue_limit = int( - self.config.default_job_requeue_limit + self.config.queue.default_job_requeue_limit ) def redis_client(self): diff --git a/src/fq/config.py b/src/fq/config.py index c3ad6b8..7f93879 100644 --- a/src/fq/config.py +++ b/src/fq/config.py @@ -13,7 +13,6 @@ @dataclass(frozen=True) class RedisConfig: - key_prefix: str conn_type: str db: int host: str | None = None @@ -29,7 +28,6 @@ def from_mapping(cls, config): cls._validate_optional(config) return cls( - key_prefix=config["key_prefix"], conn_type=config["conn_type"], db=config["db"], host=config.get("host"), @@ -41,12 +39,6 @@ def from_mapping(cls, config): @classmethod def _validate_required(cls, config): - key_prefix = cls._require_value(config, "key_prefix") - if not cls._is_non_empty_string(key_prefix): - raise FQException( - "Invalid config: redis.key_prefix must be a non-empty string" - ) - conn_type = cls._require_value(config, "conn_type") if conn_type not in REDIS_CONN_TYPES: raise FQException( @@ -118,31 +110,77 @@ def _is_int_not_bool(value): @dataclass(frozen=True) -class FQConfig: - redis: RedisConfig +class QueueConfig: + key_prefix: str job_expire_interval: int job_requeue_interval: int default_job_requeue_limit: int + @classmethod + def from_mapping(cls, config): + cls._validate_required(config) + + return cls( + key_prefix=config["key_prefix"], + job_expire_interval=config["job_expire_interval"], + job_requeue_interval=config["job_requeue_interval"], + default_job_requeue_limit=config["default_job_requeue_limit"], + ) + + @classmethod + def _validate_required(cls, config): + key_prefix = cls._require_value(config, "key_prefix") + if not cls._is_non_empty_string(key_prefix): + raise FQException( + "Invalid config: queue.key_prefix must be a non-empty string" + ) + + for option_name in ("job_expire_interval", "job_requeue_interval"): + value = cls._require_value(config, option_name) + if not is_valid_interval(value): + raise FQException( + "Invalid config: queue.%s must be a positive integer" + % option_name + ) + + default_requeue_limit = cls._require_value(config, "default_job_requeue_limit") + if not is_valid_requeue_limit(default_requeue_limit): + raise FQException( + "Invalid config: " + "queue.default_job_requeue_limit must be an integer >= -1" + ) + + @staticmethod + def _require_value(config, option_name): + if option_name not in config: + raise FQException("Missing config: queue.%s" % option_name) + + return config[option_name] + + @staticmethod + def _is_non_empty_string(value): + return isinstance(value, str) and bool(value) + + +@dataclass(frozen=True) +class FQConfig: + redis: RedisConfig + queue: QueueConfig + @classmethod def from_mapping(cls, config): normalized = cls._normalize_sections(config) cls._require_sections(normalized) - fq_config = normalized["fq"] - cls._validate_fq_section(fq_config) - return cls( redis=RedisConfig.from_mapping(normalized["redis"]), - job_expire_interval=fq_config["job_expire_interval"], - job_requeue_interval=fq_config["job_requeue_interval"], - default_job_requeue_limit=fq_config["default_job_requeue_limit"], + queue=QueueConfig.from_mapping(normalized["queue"]), ) @staticmethod def _normalize_sections(config): if not isinstance(config, Mapping): - raise FQException("Config must be a mapping with redis and fq sections") + raise FQException("Config must be a mapping with redis and queue sections") normalized = {} for section_name, section_values in config.items(): @@ -159,29 +197,5 @@ def _normalize_sections(config): @staticmethod def _require_sections(config): - if "redis" not in config or "fq" not in config: - raise FQException("Config missing required sections: redis, fq") - - @classmethod - def _validate_fq_section(cls, config): - for option_name in ("job_expire_interval", "job_requeue_interval"): - value = cls._require_value(config, option_name) - if not is_valid_interval(value): - raise FQException( - "Invalid config: fq.%s must be a positive integer" % option_name - ) - - default_requeue_limit = cls._require_value( - config, "default_job_requeue_limit" - ) - if not is_valid_requeue_limit(default_requeue_limit): - raise FQException( - "Invalid config: fq.default_job_requeue_limit must be an integer >= -1" - ) - - @staticmethod - def _require_value(config, option_name): - if option_name not in config: - raise FQException("Missing config: fq.%s" % option_name) - - return config[option_name] + if "redis" not in config or "queue" not in config: + raise FQException("Config missing required sections: redis, queue") diff --git a/tests/config.py b/tests/config.py index 83e1e6b..9d05648 100644 --- a/tests/config.py +++ b/tests/config.py @@ -9,14 +9,14 @@ TEST_CONFIG = { - "fq": { + "queue": { + "key_prefix": "test_fq", "job_expire_interval": 5000, "job_requeue_interval": 5000, "default_job_requeue_limit": -1, }, "redis": { "db": 0, - "key_prefix": "test_fq", "conn_type": "tcp_sock", "unix_socket_path": TEST_UNIX_SOCKET_PATH, "port": 6379, diff --git a/tests/test_edge_cases.py b/tests/test_edge_cases.py index 0095bf1..1400a31 100644 --- a/tests/test_edge_cases.py +++ b/tests/test_edge_cases.py @@ -150,6 +150,22 @@ def test_invalid_config_type_raises(self): with self.assertRaisesRegex(FQException, "Config must be a mapping"): FQ("does-not-exist.conf") + def test_missing_required_config_section_raises(self): + config = build_test_config() + del config["queue"] + with self.assertRaisesRegex( + FQException, "Config missing required sections: redis, queue" + ): + FQ(config) + + def test_fq_config_section_is_not_supported(self): + config = build_test_config() + config["fq"] = config.pop("queue") + with self.assertRaisesRegex( + FQException, "Config missing required sections: redis, queue" + ): + FQ(config) + async def test_initialize_fails_fast_on_bad_redis(self): with patch("fq.redis.AsyncRedis", FakeRedisConnectionFailure): fq = FQ(self.config) @@ -159,8 +175,8 @@ async def test_initialize_fails_fast_on_bad_redis(self): async def test_cluster_initialization(self): """Covers clustered Redis path (queue.py lines 69-75, 104-106).""" config = build_test_config( + queue={"key_prefix": "test_fq_cluster"}, redis={ - "key_prefix": "test_fq_cluster", "clustered": True, "password": "cluster-password", } @@ -196,15 +212,15 @@ def test_unix_socket_clustered_config_must_be_boolean(self): def test_missing_required_config_key_raises_with_path(self): config = build_test_config() - del config["redis"]["key_prefix"] - with self.assertRaisesRegex(FQException, "Missing config: redis.key_prefix"): + del config["queue"]["key_prefix"] + with self.assertRaisesRegex(FQException, "Missing config: queue.key_prefix"): FQ(config) def test_invalid_config_value_raises_with_path(self): - config = build_test_config(fq={"job_expire_interval": "5000"}) + config = build_test_config(queue={"job_expire_interval": "5000"}) with self.assertRaisesRegex( FQException, - "Invalid config: fq.job_expire_interval must be a positive integer", + "Invalid config: queue.job_expire_interval must be a positive integer", ): FQ(config) @@ -253,7 +269,7 @@ async def test_deep_status_calls_set(self): self.assertEqual( fq._r.key_set, ( - "fq:deep_status:{}".format(fq.config.redis.key_prefix), + "fq:deep_status:{}".format(fq.config.queue.key_prefix), "sharq_deep_status", ), ) diff --git a/tests/test_func.py b/tests/test_func.py index 373d491..a2411ce 100644 --- a/tests/test_func.py +++ b/tests/test_func.py @@ -1811,8 +1811,8 @@ async def test_close_with_none_client(self): async def test_initialize_unix_socket_connection(self): """Test initialization with Unix socket connection - tests line 59.""" config = build_test_config( + queue={"key_prefix": "test_fq_unix"}, redis={ - "key_prefix": "test_fq_unix", "conn_type": "unix_sock", "unix_socket_path": NONEXISTENT_UNIX_SOCKET_PATH, } diff --git a/tests/test_sync_queue.py b/tests/test_sync_queue.py index d3dd284..2ae557e 100644 --- a/tests/test_sync_queue.py +++ b/tests/test_sync_queue.py @@ -14,7 +14,7 @@ class SyncFQTest(unittest.TestCase): def setUp(self): - self.queue = FQ(build_test_config(redis={"key_prefix": "test_fq_sync"})) + self.queue = FQ(build_test_config(queue={"key_prefix": "test_fq_sync"})) self.queue.initialize() self.queue._r.flushdb() self.queue_type = "sms" @@ -94,8 +94,10 @@ def test_requeue_behavior(self): self.queue.close() self.queue = FQ( build_test_config( - fq={"job_expire_interval": 20}, - redis={"key_prefix": "test_fq_sync_requeue"}, + queue={ + "job_expire_interval": 20, + "key_prefix": "test_fq_sync_requeue", + }, ) ) self.queue.initialize() @@ -245,7 +247,7 @@ def collect_sync_errors(): return errors async def collect_async_errors(): - queue = AsyncFQ(build_test_config(redis={"key_prefix": "test_fq_async"})) + queue = AsyncFQ(build_test_config(queue={"key_prefix": "test_fq_async"})) await queue.initialize() await queue._r.flushdb() checks = [ @@ -289,7 +291,7 @@ async def collect_async_errors(): def test_sync_async_interoperability(self): async def scenario(): - config = build_test_config(redis={"key_prefix": "test_fq_sync_interop"}) + config = build_test_config(queue={"key_prefix": "test_fq_sync_interop"}) async_queue = AsyncFQ(config) sync_queue = FQ(config) From c3c2bb73e2b64df458ec77e779465079d012e5ce Mon Sep 17 00:00:00 2001 From: "Ochui, Princewill Patrick" <21917688+ochui@users.noreply.github.com> Date: Wed, 6 May 2026 21:46:51 +0100 Subject: [PATCH 12/12] Improves error chaining and clarifies Redis socket config Adds exception chaining when payload serialization fails to improve debugging. Updates documentation to clarify usage of Redis Unix sockets and example config. Enhances test to assert exception cause, ensuring better error traceability. --- README.md | 19 ++++++++++++++----- src/fq/validators.py | 4 ++-- tests/test_queue.py | 5 ++++- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 32fa728..4eeb71d 100644 --- a/README.md +++ b/README.md @@ -45,17 +45,28 @@ config = { }, "redis": { "db": 0, - "conn_type": "tcp_sock", # or "unix_sock" + "conn_type": "tcp_sock", "host": "127.0.0.1", "port": 6379, "password": "", "clustered": False, - "unix_socket_path": "/tmp/redis.sock", }, } ``` -> If you connect via Unix sockets, uncomment the `unixsocket` lines in your `redis.conf`: +For Unix socket connections, use `conn_type: "unix_sock"` and provide +`unix_socket_path`: +```python +"redis": { + "db": 0, + "conn_type": "unix_sock", + "unix_socket_path": "/tmp/redis.sock", + "password": "", + "clustered": False, +} +``` + +> If you use Unix sockets, uncomment the `unixsocket` lines in your `redis.conf`: > ``` > unixsocket /var/run/redis/redis.sock > unixsocketperm 755 @@ -90,7 +101,6 @@ async def main(): "port": 6379, "password": "", "clustered": False, - "unix_socket_path": "/tmp/redis.sock", }, } @@ -144,7 +154,6 @@ config = { "port": 6379, "password": "", "clustered": False, - "unix_socket_path": "/tmp/redis.sock", }, } diff --git a/src/fq/validators.py b/src/fq/validators.py index 5e4913e..5b869dd 100644 --- a/src/fq/validators.py +++ b/src/fq/validators.py @@ -49,8 +49,8 @@ def validate_enqueue_arguments( try: serialized_payload = serialize_payload(payload) - except TypeError: - raise BadArgumentException("can not serialize.") + except TypeError as exc: + raise BadArgumentException("can not serialize.") from exc return EnqueueArguments( serialized_payload=serialized_payload, diff --git a/tests/test_queue.py b/tests/test_queue.py index 1f5c1d6..e672c68 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -324,7 +324,9 @@ async def test_enqueue_requeue_limit_invalid(self): ) async def test_enqueue_cannot_serialize_payload(self): - with self.assertRaisesRegex(BadArgumentException, r"can not serialize."): + with self.assertRaisesRegex( + BadArgumentException, r"can not serialize." + ) as ctx: await self.queue.enqueue( payload=self.invalid_payload, interval=self.valid_interval, @@ -332,6 +334,7 @@ async def test_enqueue_cannot_serialize_payload(self): queue_id=self.valid_queue_id, queue_type=self.valid_queue_type, ) + self.assertIsInstance(ctx.exception.__cause__, TypeError) async def test_enqueue_all_ok(self): # with a queue_type