Skip to content

Commit ae27f17

Browse files
committed
fix: address code review findings in invalidate/cleanup-snapshots
- C1/M4: eliminate TOCTOU race in delete_snapshots_for_environment by calling state_sync.delete_snapshots(batch.expired_snapshot_ids) directly instead of re-querying via delete_expired_snapshots, so physical drops and state removal operate on the same snapshot ID set - M1: remove always-truthy `if target_conditions:` guard in get_expired_snapshots (snapshot_id_filter always yields >= 1 condition) - M2: when cleanup_snapshots=True and the environment does not exist, log a warning and return early instead of printing a misleading success message - m1: unconditionally initialize target_snapshot_ids before the cleanup_snapshots block to prevent potential UnboundLocalError - n1: enforce `sync = sync or cleanup_snapshots` explicitly so the implication is in code, not just docs; update docstring and CLI help to say "cleanup runs synchronously" instead of "Implies --sync"
1 parent 2e23f5f commit ae27f17

4 files changed

Lines changed: 13 additions & 11 deletions

File tree

sqlmesh/cli/main.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -623,7 +623,7 @@ def run(ctx: click.Context, environment: t.Optional[str] = None, **kwargs: t.Any
623623
@click.option(
624624
"--cleanup-snapshots",
625625
is_flag=True,
626-
help="After invalidating, immediately delete physical snapshot tables that are exclusively owned by this environment (not referenced by any other environment). Implies --sync.",
626+
help="After invalidating, immediately delete physical snapshot tables that are exclusively owned by this environment (not referenced by any other environment). Cleanup runs synchronously regardless of --sync.",
627627
)
628628
@click.pass_context
629629
@error_handler

sqlmesh/core/context.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,7 @@
9999
Snapshot,
100100
SnapshotEvaluator,
101101
SnapshotFingerprint,
102+
SnapshotId,
102103
missing_intervals,
103104
to_table_mapping,
104105
)
@@ -1849,18 +1850,24 @@ def invalidate_environment(
18491850
sync: If True, the call blocks until the environment is deleted. Otherwise, the environment will
18501851
be deleted asynchronously by the janitor process.
18511852
cleanup_snapshots: If True, immediately deletes physical snapshot tables that are exclusively
1852-
owned by this environment (not referenced by any other environment). Implies sync=True.
1853+
owned by this environment (not referenced by any other environment). Cleanup runs
1854+
synchronously regardless of --sync.
18531855
"""
18541856
name = Environment.sanitize_name(name)
1857+
sync = sync or cleanup_snapshots
18551858

1859+
target_snapshot_ids: t.Set[SnapshotId] = set()
18561860
if cleanup_snapshots:
18571861
# Capture snapshot IDs before invalidation so we can scope the cleanup afterwards.
18581862
env = self.state_sync.get_environment(name)
1859-
target_snapshot_ids = {s.snapshot_id for s in env.snapshots} if env else set()
1863+
if env is None:
1864+
logger.warning("Environment '%s' does not exist; skipping snapshot cleanup.", name)
1865+
return
1866+
target_snapshot_ids = {s.snapshot_id for s in env.snapshots}
18601867

18611868
self.state_sync.invalidate_environment(name)
18621869

1863-
if sync or cleanup_snapshots:
1870+
if sync:
18641871
self._cleanup_environments(name=name)
18651872
if cleanup_snapshots and target_snapshot_ids:
18661873
failures = delete_snapshots_for_environment(

sqlmesh/core/janitor.py

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -251,11 +251,7 @@ def delete_snapshots_for_environment(
251251

252252
if cleanup_succeeded or force_delete:
253253
try:
254-
state_sync.delete_expired_snapshots(
255-
batch_range=ExpiredBatchRange.all_batch_range(),
256-
ignore_ttl=True,
257-
target_snapshot_ids=target_snapshot_ids,
258-
)
254+
state_sync.delete_snapshots(batch.expired_snapshot_ids)
259255
logger.info(
260256
"Cleaned up %s snapshots from invalidated environment",
261257
len(batch.expired_snapshot_ids),

sqlmesh/core/state_sync/db/snapshot.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,7 @@ def get_expired_snapshots(
189189
batch_size=self.SNAPSHOT_BATCH_SIZE,
190190
)
191191
)
192-
if target_conditions:
193-
expired_query = expired_query.where(exp.or_(*target_conditions))
192+
expired_query = expired_query.where(exp.or_(*target_conditions))
194193

195194
expired_query = expired_query.where(batch_range.where_filter)
196195

0 commit comments

Comments
 (0)