From fb145ecc40fef00215d1cf112cad14adbdf63f8d Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 20 Oct 2026 19:15:23 +0000 Subject: [PATCH 1/3] stop sharing the default request queue between crawlers --- docs/upgrading/upgrading_to_v2.md | 12 ++++++++++ src/crawlee/crawlers/_basic/_basic_crawler.py | 4 +++- .../crawlers/_basic/test_basic_crawler.py | 23 +++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 894d2a8567..c3b64a2f3b 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -167,3 +167,15 @@ current concurrency 3, desired 5, overloaded: cpu 58% of the time (limit 40%) To read the concurrency values in code, use `crawler.concurrency_system`. The autoscaling loggers moved together with the package. `crawlee._autoscaling.autoscaled_pool`, `crawlee._autoscaling.snapshotter` and `crawlee._autoscaling.system_status` are now `crawlee.autoscaling._autoscaled_pool`, `crawlee.autoscaling._snapshotter` and `crawlee.autoscaling._system_status`. The state log, along with the messages about scaling and free capacity, now comes from the new `crawlee.autoscaling._concurrency_system` logger. To configure all of them at once, use the `crawlee.autoscaling` logger. + +## Only the first crawler uses the default request queue + +Only the first crawler created in a process opens the default request queue, the same as Crawlee JS. Every later crawler without a `request_manager` opens its own queue under the alias `__default_{id}__`. A crawler without an `id` uses its creation index there, such as `__default_1__`. In v1, all these crawlers shared the default queue, so a URL one crawler handled was skipped by the others. + +To reach a crawler's queue, call `crawler.get_request_manager()` instead of `RequestQueue.open()`. If your crawlers relied on the shared queue, open it yourself and pass it to each of them: + +```python +queue = await RequestQueue.open() +crawler_1 = BasicCrawler(request_manager=queue) +crawler_2 = BasicCrawler(request_manager=queue) +``` diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index c7af459092..74bc72aa32 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -381,6 +381,7 @@ def __init__( Intended for use by subclasses rather than direct instantiation of `BasicCrawler`. """ self._id = str(BasicCrawler.__next_id) if id is None else id + self._default_rq_alias = None if BasicCrawler.__next_id == 0 else f'__default_{self._id}__' BasicCrawler.__next_id += 1 self._consumer = ConcurrencyConsumer(id=self._id) @@ -644,9 +645,10 @@ async def _get_proxy_info(self, request: Request, session: Session | None) -> Pr ) async def get_request_manager(self) -> RequestManager: - """Return the configured request manager. If none is configured, open and return the default request queue.""" + """Return the configured request manager. If none is configured, open and return the crawler's own queue.""" if not self._request_manager: self._request_manager = await RequestQueue.open( + alias=self._default_rq_alias, storage_client=self._service_locator.get_storage_client(), configuration=self._service_locator.get_configuration(), ) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 3ef451b259..f062e63020 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -1270,6 +1270,29 @@ def test_default_statistics_key_differs_from_later_statistics() -> None: assert statistics._state._persist_state_key != crawler.statistics._state._persist_state_key +async def test_only_first_crawler_uses_default_request_queue() -> None: + """Only the first crawler opens the default RQ, the later ones open their own under an alias from their ID.""" + crawler_1 = BasicCrawler(id='first') + crawler_2 = BasicCrawler() + crawler_3 = BasicCrawler(id='custom') + + assert await crawler_1.get_request_manager() is await RequestQueue.open() + assert await crawler_2.get_request_manager() is await RequestQueue.open(alias='__default_1__') + assert await crawler_3.get_request_manager() is await RequestQueue.open(alias='__default_custom__') + + +async def test_crawlers_do_not_share_requests() -> None: + """Two crawlers without a request manager each crawl the same URL, since they don't share a queue.""" + crawler_1 = BasicCrawler(request_handler=AsyncMock()) + crawler_2 = BasicCrawler(request_handler=AsyncMock()) + + stats_1 = await crawler_1.run(['https://a.placeholder.com']) + stats_2 = await crawler_2.run(['https://a.placeholder.com']) + + assert stats_1.requests_finished == 1 + assert stats_2.requests_finished == 1 + + async def test_context_handlers_use_state(key_value_store: KeyValueStore) -> None: state_in_handler_one: dict[str, JsonSerializable] = {} state_in_handler_two: dict[str, JsonSerializable] = {} From 499ceafcdb049b321c5a4fdda4e6364bb3c3900a Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 21 Oct 2026 15:36:19 +0300 Subject: [PATCH 2/3] Apply batched suggestions from code review Co-authored-by: Vlada Dusek --- src/crawlee/crawlers/_basic/_basic_crawler.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 74bc72aa32..fd4bcee6b3 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -224,8 +224,9 @@ class _BasicCrawlerOptions(TypedDict): Returning `None` suppresses the status message.""" id: NotRequired[str] - """A unique identifier for the crawler instance. It isolates the `use_state` state and the default statistics from - other crawlers. Without it, the ID follows the order in which crawlers are created.""" + """Identifier of the crawler. Crawlers with the same ID share the `use_state` state and the default statistics, and + every crawler but the first uses it in the alias of its default request queue. Without it, the ID follows the order + in which crawlers are created.""" class _BasicCrawlerOptionsGeneric(TypedDict, Generic[TCrawlingContext, TStatisticsState]): @@ -371,8 +372,9 @@ def __init__( status_message_logging_interval: Interval for logging the crawler status messages. status_message_callback: Allows overriding the default status message. The default status message is provided in the parameters. Returning `None` suppresses the status message. - id: A unique identifier for the crawler instance. It isolates the `use_state` state and the default - statistics from other crawlers. Without it, the ID follows the order in which crawlers are created. + id: Identifier of the crawler. Crawlers with the same ID share the `use_state` state and the default + statistics, and every crawler but the first uses it in the alias of its default request queue. Without + it, the ID follows the order in which crawlers are created. _context_pipeline: Enables extending the request lifecycle and modifying the crawling context. Intended for use by subclasses rather than direct instantiation of `BasicCrawler`. _additional_context_managers: Additional context managers used throughout the crawler lifecycle. From 2591a476df6447924dad114043ac54c1905cbd75 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 21 Oct 2026 12:53:13 +0000 Subject: [PATCH 3/3] add test --- tests/unit/crawlers/_basic/test_basic_crawler.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index f062e63020..422b0aac37 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -1293,6 +1293,16 @@ async def test_crawlers_do_not_share_requests() -> None: assert stats_2.requests_finished == 1 +async def test_id_invalid_as_request_queue_alias() -> None: + """A later crawler whose ID can't form an RQ alias fails to open its RQ, unless it gets a request manager.""" + BasicCrawler() + with pytest.raises(ValueError, match='Invalid storage alias'): + await BasicCrawler(id='shop/products').get_request_manager() + request_manager = Mock() + crawler = BasicCrawler(id='shop/products', request_manager=request_manager) + assert await crawler.get_request_manager() is request_manager + + async def test_context_handlers_use_state(key_value_store: KeyValueStore) -> None: state_in_handler_one: dict[str, JsonSerializable] = {} state_in_handler_two: dict[str, JsonSerializable] = {}