Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/upgrading/upgrading_to_v2.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@ title: Upgrading to v2

This page summarizes the breaking changes between Crawlee for Python v1 and v2.

## Session retirement is permanent

`Session.retire()` permanently blocks the session. Calling `mark_good()` can still lower its error score, but cannot make it usable again. Create a new session to replace a retired one. Repeated calls to `retire()` leave the error score unchanged.

Session state includes a `retired` flag, which is preserved when the session pool is saved and restored. Saved sessions without this field load with `retired=False` and retain their existing error scores and usage limits.

## Python version support

We drop support for Python 3.10, which reached its end of life in October 2026. The minimum supported version is now Python 3.11.
Expand Down
1 change: 1 addition & 0 deletions src/crawlee/sessions/_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ class SessionModel(BaseModel):
usage_count: int
max_usage_count: int
error_score: float
retired: bool = False
cookies: list[CookieParam]
blocked_status_codes: list[int]

Expand Down
14 changes: 10 additions & 4 deletions src/crawlee/sessions/_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ def __init__(
usage_count: int = 0,
max_usage_count: int = 50,
error_score: float = 0.0,
retired: bool = False,
cookies: SessionCookies | CookieJar | dict[str, str] | list[CookieParam] | None = None,
blocked_status_codes: list | None = None,
) -> None:
Expand All @@ -60,6 +61,7 @@ def __init__(
usage_count: Number of times the session has been used.
max_usage_count: Maximum allowable uses of the session before it is considered expired.
error_score: Current error score of the session.
retired: Whether the session has been permanently retired.
cookies: Cookies associated with the session.
blocked_status_codes: HTTP status codes that indicate a session should be blocked.
"""
Expand All @@ -72,6 +74,7 @@ def __init__(
self._usage_count = usage_count
self._max_usage_count = max_usage_count
self._error_score = error_score
self._retired = retired
self._cookies = SessionCookies(cookies) or SessionCookies()
self._blocked_status_codes = set(blocked_status_codes or self._DEFAULT_BLOCKED_STATUS_CODES)

Expand Down Expand Up @@ -145,8 +148,8 @@ def expires_at(self) -> datetime:

@property
def is_blocked(self) -> bool:
"""Indicate whether the session is blocked based on the error score.."""
return self._error_score >= self._max_error_score
"""Indicate whether the session is retired or blocked by its error score."""
return self._retired or self._error_score >= self._max_error_score

@property
def is_expired(self) -> bool:
Expand Down Expand Up @@ -183,6 +186,7 @@ def get_state(self, *, as_dict: bool = False) -> SessionModel | dict:
usage_count=self._usage_count,
max_usage_count=self._max_usage_count,
error_score=self._error_score,
retired=self._retired,
cookies=self._cookies.get_cookies_as_dicts(),
blocked_status_codes=list(self._blocked_status_codes),
)
Expand Down Expand Up @@ -211,14 +215,16 @@ def mark_bad(self) -> None:
self.retire()

def retire(self) -> None:
"""Retire the session by setting the error score to the maximum value.
"""Permanently retire the session and increase its error score once.

This method should be used if the session usage was unsuccessful and you are sure that it is because of
the session configuration and not any external matters. For example when server returns 403 status code.
If the session does not work due to some external factors as server error such as 5XX you probably want
to use `mark_bad` method.
"""
self._error_score += self._max_error_score
if not self._retired:
self._retired = True
self._error_score += self._max_error_score

def is_blocked_status_code(
self,
Expand Down
44 changes: 44 additions & 0 deletions tests/unit/sessions/test_session.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import pytest

from crawlee.sessions._cookies import SessionCookies
from crawlee.sessions._models import SessionModel
from crawlee.sessions._session import Session


Expand Down Expand Up @@ -89,6 +90,49 @@ def test_retire_method(session: Session) -> None:
assert session.usage_count == initial_usage_count


def test_retired_session_stays_blocked_after_success() -> None:
"""A successful request cannot make a retired session usable again."""
session = Session()
session.retire()
session.mark_good()

assert session.error_score < 3.0
assert session.is_blocked
assert not session.is_usable


def test_retirement_survives_serialization() -> None:
"""Retirement survives a JSON round trip even with a reduced error score."""
session = Session()
session.retire()
session.mark_good()
state = session.get_state(as_dict=False).model_dump_json(by_alias=True)
restored = Session.from_model(SessionModel.model_validate_json(state))
restored.mark_good()

assert restored.is_blocked
assert not restored.is_usable


def test_session_without_retirement_field_can_be_restored() -> None:
"""Saved sessions without a retirement field remain loadable."""
state = Session().get_state(as_dict=True)
state.pop('retired', None)
restored = Session.from_model(SessionModel.model_validate(state))

assert restored.is_usable


def test_retire_is_idempotent() -> None:
"""Repeated retirement leaves the error score and usage count unchanged."""
session = Session()
session.retire()
state = session.get_state(as_dict=True)
session.retire()

assert session.get_state(as_dict=True) == state


def test_mark_good_at_usage_limit_no_double_increment() -> None:
"""Test that mark_good at max usage count does not double-increment usage_count via retire."""
session = Session(max_usage_count=5, usage_count=4)
Expand Down
46 changes: 46 additions & 0 deletions tests/unit/sessions/test_session_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,19 @@ async def test_get_session(session_pool: SessionPool) -> None:
assert session.is_usable


async def test_pool_does_not_reuse_retired_session_after_success() -> None:
"""Neither pool lookup returns a retired session after a successful request."""
async with SessionPool(max_pool_size=1, persistence_enabled=False) as pool:
session = await pool.get_session()
session.retire()
session.mark_good()

assert await pool.get_session_by_id(session.id) is None
replacement = await pool.get_session()
assert replacement.id != session.id
assert replacement.is_usable


async def test_get_session_no_usable(caplog: pytest.LogCaptureFixture, session_pool: SessionPool) -> None:
"""Ensure that retrieval of a non-existent or retired session returns None and logs warning."""
session = await session_pool.get_session_by_id('non_existent')
Expand Down Expand Up @@ -170,6 +183,39 @@ async def test_session_pool_persist_and_restore(event_manager: EventManager, kvs
assert previous_state is None


async def test_retired_session_is_not_restored(event_manager: EventManager, kvs: KeyValueStore) -> None:
"""A saved retired session is excluded when the pool is reopened."""
service_locator.set_event_manager(event_manager)

async with SessionPool(
max_pool_size=1,
persistence_enabled=True,
persist_state_kvs_name=KVS_NAME,
persist_state_key=PERSIST_STATE_KEY,
) as pool:
session = await pool.get_session()
session.retire()
session.mark_good()
event_manager.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=False))
await event_manager.wait_for_all_listeners_to_complete()

saved_state = await kvs.get_value(PERSIST_STATE_KEY)
assert isinstance(saved_state, dict)
saved_pool = SessionPoolModel.model_validate(saved_state)
assert not saved_pool.sessions[session.id].is_usable

async with SessionPool(
max_pool_size=1,
persistence_enabled=True,
persist_state_kvs_name=KVS_NAME,
persist_state_key=PERSIST_STATE_KEY,
) as restored_pool:
assert await restored_pool.get_session_by_id(session.id) is None
replacement = await restored_pool.get_session()
assert replacement.id != session.id
assert replacement.is_usable


async def test_methods_raise_error_when_not_active() -> None:
session = Session()
session_pool = SessionPool()
Expand Down
Loading