From f63000b3b2dd58c8572fea81624ce3e64cb120b7 Mon Sep 17 00:00:00 2001 From: Harris Effron Date: Mon, 15 Jun 2026 10:26:04 -0400 Subject: [PATCH 1/2] fix: max_age metric excludes locked jobs --- lib/delayed/monitor.rb | 9 +- .../__snapshots__/monitor_spec.rb.snap | 100 +++++++++--------- spec/delayed/monitor_spec.rb | 38 +++++++ 3 files changed, 92 insertions(+), 55 deletions(-) diff --git a/lib/delayed/monitor.rb b/lib/delayed/monitor.rb index 1245ea9..c2871cb 100644 --- a/lib/delayed/monitor.rb +++ b/lib/delayed/monitor.rb @@ -147,11 +147,11 @@ def max_lock_age_grouped end def max_age_grouped - live_counts.transform_values { |j| time_ago(db_now(j), j.run_at) } + pending_counts.transform_values { |j| time_ago(db_now(j), j.run_at) } end def alert_age_percent_grouped - live_counts.each_with_object({}) do |((priority, queue), j), metrics| + pending_counts.each_with_object({}) do |((priority, queue), j), metrics| max_age = time_ago(db_now(j), j.run_at) alert_age = Priority.new(priority).alert_age metrics[[priority, queue]] = [max_age / alert_age * 100, 100].min if alert_age @@ -169,17 +169,15 @@ def oldest_locked_job_grouped end def oldest_workable_job_grouped - live_counts.transform_values(&:run_at).compact + pending_counts.transform_values(&:run_at).compact end def live_counts @memo[:live_counts] ||= grouped_query( jobs.live, - include_db_time: true, count: [:count, '*'], future_count: [:sum, case_when(Job.future_clause.to_sql)], erroring_count: [:sum, case_when(Job.erroring_clause.to_sql)], - run_at: [:min, case_when(Job.pending_clause.to_sql, 'run_at')], ) end @@ -190,6 +188,7 @@ def pending_counts claimed_count: [:sum, case_when(Job.claimed_clause.to_sql)], claimable_count: [:sum, case_when(Job.claimable_clause.to_sql)], locked_at: [:min, case_when(Job.claimed_clause.to_sql, 'locked_at')], + run_at: [:min, case_when(Job.claimable_clause.to_sql, 'run_at')], ) end diff --git a/spec/delayed/__snapshots__/monitor_spec.rb.snap b/spec/delayed/__snapshots__/monitor_spec.rb.snap index 0a9b802..8da11e2 100644 --- a/spec/delayed/__snapshots__/monitor_spec.rb.snap +++ b/spec/delayed/__snapshots__/monitor_spec.rb.snap @@ -27,29 +27,26 @@ GroupAggregate (cost=...) SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, - MIN(run_at) AS run_at, - TIMEZONE('UTC', STATEMENT_TIMESTAMP()) AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", COUNT(*) AS count, SUM(CASE WHEN \"delayed_jobs\".\"run_at\" > '2025-11-10 17:20:13' THEN 1 ELSE 0 END) AS future_count, - SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count, - MIN(CASE WHEN \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' THEN run_at ELSE NULL END) AS run_at + SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL GROUP BY \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\") subquery GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, \"queue\" GroupAggregate (cost=...) - Output: sum(subquery.count), sum(subquery.future_count), sum(subquery.erroring_count), min(subquery.run_at), timezone('UTC'::text, statement_timestamp()), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue + Output: sum(subquery.count), sum(subquery.future_count), sum(subquery.erroring_count), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue Group Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Sort (cost=...) - Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count, subquery.run_at + Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count Sort Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Subquery Scan on subquery (cost=...) - Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count, subquery.run_at + Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count -> GroupAggregate (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, count(*), sum(CASE WHEN (delayed_jobs.run_at > '2025-11-10 17:20:13'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN (delayed_jobs.attempts > 0) THEN 1 ELSE 0 END), min(CASE WHEN (delayed_jobs.run_at <= '2025-11-10 17:20:13'::timestamp without time zone) THEN delayed_jobs.run_at ELSE NULL::timestamp without time zone END) + Output: delayed_jobs.priority, delayed_jobs.queue, count(*), sum(CASE WHEN (delayed_jobs.run_at > '2025-11-10 17:20:13'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN (delayed_jobs.attempts > 0) THEN 1 ELSE 0 END) Group Key: delayed_jobs.priority, delayed_jobs.queue -> Sort (cost=...) Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.run_at, delayed_jobs.attempts @@ -65,13 +62,16 @@ GroupAggregate (cost=...) SELECT SUM(claimed_count) AS claimed_count, SUM(claimable_count) AS claimable_count, MIN(locked_at) AS locked_at, + MIN(run_at) AS run_at, TIMEZONE('UTC', STATEMENT_TIMESTAMP()) AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", SUM(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN 1 ELSE 0 END) AS claimed_count, SUM(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN 1 ELSE 0 END) AS claimable_count, - MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at + MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at, + MIN(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL + OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN run_at ELSE NULL END) AS run_at FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL AND \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' @@ -79,21 +79,21 @@ SELECT SUM(claimed_count) AS claimed_count, GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, \"queue\" GroupAggregate (cost=...) - Output: sum(subquery.claimed_count), sum(subquery.claimable_count), min(subquery.locked_at), timezone('UTC'::text, statement_timestamp()), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue + Output: sum(subquery.claimed_count), sum(subquery.claimable_count), min(subquery.locked_at), min(subquery.run_at), timezone('UTC'::text, statement_timestamp()), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue Group Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Sort (cost=...) - Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at + Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at, subquery.run_at Sort Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Subquery Scan on subquery (cost=...) - Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at + Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at, subquery.run_at -> GroupAggregate (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, sum(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN ((delayed_jobs.locked_at IS NULL) OR (delayed_jobs.locked_at < '2025-11-10 16:59:43'::timestamp without time zone)) THEN 1 ELSE 0 END), min(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN delayed_jobs.locked_at ELSE NULL::timestamp without time zone END) + Output: delayed_jobs.priority, delayed_jobs.queue, sum(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN ((delayed_jobs.locked_at IS NULL) OR (delayed_jobs.locked_at < '2025-11-10 16:59:43'::timestamp without time zone)) THEN 1 ELSE 0 END), min(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN delayed_jobs.locked_at ELSE NULL::timestamp without time zone END), min(CASE WHEN ((delayed_jobs.locked_at IS NULL) OR (delayed_jobs.locked_at < '2025-11-10 16:59:43'::timestamp without time zone)) THEN delayed_jobs.run_at ELSE NULL::timestamp without time zone END) Group Key: delayed_jobs.priority, delayed_jobs.queue -> Sort (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at + Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at, delayed_jobs.run_at Sort Key: delayed_jobs.priority, delayed_jobs.queue -> Index Scan using idx_delayed_jobs_live on public.delayed_jobs (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at + Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at, delayed_jobs.run_at Index Cond: (delayed_jobs.run_at <= '2025-11-10 17:20:13'::timestamp without time zone) --- -- QUERIES FOR `erroring_count`: @@ -152,29 +152,26 @@ GroupAggregate (cost=...) SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, - MIN(run_at) AS run_at, - TIMEZONE('UTC', STATEMENT_TIMESTAMP()) AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", COUNT(*) AS count, SUM(CASE WHEN \"delayed_jobs\".\"run_at\" > '2025-11-10 17:20:13' THEN 1 ELSE 0 END) AS future_count, - SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count, - MIN(CASE WHEN \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' THEN run_at ELSE NULL END) AS run_at + SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL GROUP BY \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\") subquery GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, \"queue\" GroupAggregate (cost=...) - Output: sum(subquery.count), sum(subquery.future_count), sum(subquery.erroring_count), min(subquery.run_at), timezone('UTC'::text, statement_timestamp()), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue + Output: sum(subquery.count), sum(subquery.future_count), sum(subquery.erroring_count), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue Group Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Sort (cost=...) - Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count, subquery.run_at + Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count Sort Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Subquery Scan on subquery (cost=...) - Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count, subquery.run_at + Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.count, subquery.future_count, subquery.erroring_count -> GroupAggregate (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, count(*), sum(CASE WHEN (delayed_jobs.run_at > '2025-11-10 17:20:13'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN (delayed_jobs.attempts > 0) THEN 1 ELSE 0 END), min(CASE WHEN (delayed_jobs.run_at <= '2025-11-10 17:20:13'::timestamp without time zone) THEN delayed_jobs.run_at ELSE NULL::timestamp without time zone END) + Output: delayed_jobs.priority, delayed_jobs.queue, count(*), sum(CASE WHEN (delayed_jobs.run_at > '2025-11-10 17:20:13'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN (delayed_jobs.attempts > 0) THEN 1 ELSE 0 END) Group Key: delayed_jobs.priority, delayed_jobs.queue -> Sort (cost=...) Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.run_at, delayed_jobs.attempts @@ -191,13 +188,16 @@ GroupAggregate (cost=...) SELECT SUM(claimed_count) AS claimed_count, SUM(claimable_count) AS claimable_count, MIN(locked_at) AS locked_at, + MIN(run_at) AS run_at, TIMEZONE('UTC', STATEMENT_TIMESTAMP()) AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", SUM(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN 1 ELSE 0 END) AS claimed_count, SUM(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN 1 ELSE 0 END) AS claimable_count, - MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at + MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at, + MIN(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL + OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN run_at ELSE NULL END) AS run_at FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL AND \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' @@ -205,21 +205,21 @@ SELECT SUM(claimed_count) AS claimed_count, GROUP BY CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END, \"queue\" GroupAggregate (cost=...) - Output: sum(subquery.claimed_count), sum(subquery.claimable_count), min(subquery.locked_at), timezone('UTC'::text, statement_timestamp()), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue + Output: sum(subquery.claimed_count), sum(subquery.claimable_count), min(subquery.locked_at), min(subquery.run_at), timezone('UTC'::text, statement_timestamp()), (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue Group Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Sort (cost=...) - Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at + Output: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at, subquery.run_at Sort Key: (CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END), subquery.queue -> Subquery Scan on subquery (cost=...) - Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at + Output: CASE WHEN (subquery.priority < 10) THEN 0 WHEN (subquery.priority < 20) THEN 10 WHEN (subquery.priority < 30) THEN 20 WHEN (subquery.priority >= 30) THEN 30 ELSE NULL::integer END, subquery.queue, subquery.claimed_count, subquery.claimable_count, subquery.locked_at, subquery.run_at -> GroupAggregate (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, sum(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN ((delayed_jobs.locked_at IS NULL) OR (delayed_jobs.locked_at < '2025-11-10 16:59:43'::timestamp without time zone)) THEN 1 ELSE 0 END), min(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN delayed_jobs.locked_at ELSE NULL::timestamp without time zone END) + Output: delayed_jobs.priority, delayed_jobs.queue, sum(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN 1 ELSE 0 END), sum(CASE WHEN ((delayed_jobs.locked_at IS NULL) OR (delayed_jobs.locked_at < '2025-11-10 16:59:43'::timestamp without time zone)) THEN 1 ELSE 0 END), min(CASE WHEN (delayed_jobs.locked_at >= '2025-11-10 16:59:43'::timestamp without time zone) THEN delayed_jobs.locked_at ELSE NULL::timestamp without time zone END), min(CASE WHEN ((delayed_jobs.locked_at IS NULL) OR (delayed_jobs.locked_at < '2025-11-10 16:59:43'::timestamp without time zone)) THEN delayed_jobs.run_at ELSE NULL::timestamp without time zone END) Group Key: delayed_jobs.priority, delayed_jobs.queue -> Sort (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at + Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at, delayed_jobs.run_at Sort Key: delayed_jobs.priority, delayed_jobs.queue -> Index Scan using delayed_jobs_priority on public.delayed_jobs (cost=...) - Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at + Output: delayed_jobs.priority, delayed_jobs.queue, delayed_jobs.locked_at, delayed_jobs.run_at Index Cond: (delayed_jobs.run_at <= '2025-11-10 17:20:13'::timestamp without time zone) Filter: (delayed_jobs.failed_at IS NULL) --- @@ -266,14 +266,11 @@ USE TEMP B-TREE FOR GROUP BY SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, - MIN(run_at) AS run_at, - CURRENT_TIMESTAMP AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", COUNT(*) AS count, SUM(CASE WHEN \"delayed_jobs\".\"run_at\" > '2025-11-10 17:20:13' THEN 1 ELSE 0 END) AS future_count, - SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count, - MIN(CASE WHEN \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' THEN run_at ELSE NULL END) AS run_at + SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL GROUP BY \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\") subquery @@ -293,13 +290,16 @@ USE TEMP B-TREE FOR GROUP BY SELECT SUM(claimed_count) AS claimed_count, SUM(claimable_count) AS claimable_count, MIN(locked_at) AS locked_at, + MIN(run_at) AS run_at, CURRENT_TIMESTAMP AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", SUM(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN 1 ELSE 0 END) AS claimed_count, SUM(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN 1 ELSE 0 END) AS claimable_count, - MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at + MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at, + MIN(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL + OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN run_at ELSE NULL END) AS run_at FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL AND \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' @@ -356,14 +356,11 @@ USE TEMP B-TREE FOR GROUP BY SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, - MIN(run_at) AS run_at, - CURRENT_TIMESTAMP AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", COUNT(*) AS count, SUM(CASE WHEN \"delayed_jobs\".\"run_at\" > '2025-11-10 17:20:13' THEN 1 ELSE 0 END) AS future_count, - SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count, - MIN(CASE WHEN \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' THEN run_at ELSE NULL END) AS run_at + SUM(CASE WHEN \"delayed_jobs\".\"attempts\" > 0 THEN 1 ELSE 0 END) AS erroring_count FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL GROUP BY \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\") subquery @@ -383,13 +380,16 @@ USE TEMP B-TREE FOR GROUP BY SELECT SUM(claimed_count) AS claimed_count, SUM(claimable_count) AS claimable_count, MIN(locked_at) AS locked_at, + MIN(run_at) AS run_at, CURRENT_TIMESTAMP AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT \"delayed_jobs\".\"priority\", \"delayed_jobs\".\"queue\", SUM(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN 1 ELSE 0 END) AS claimed_count, SUM(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN 1 ELSE 0 END) AS claimable_count, - MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at + MIN(CASE WHEN \"delayed_jobs\".\"locked_at\" >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at, + MIN(CASE WHEN (\"delayed_jobs\".\"locked_at\" IS NULL + OR \"delayed_jobs\".\"locked_at\" < '2025-11-10 16:59:43') THEN run_at ELSE NULL END) AS run_at FROM \"delayed_jobs\" WHERE \"delayed_jobs\".\"failed_at\" IS NULL AND \"delayed_jobs\".\"run_at\" <= '2025-11-10 17:20:13' @@ -449,14 +449,11 @@ SELECT SUM(count) AS count, SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, - MIN(run_at) AS run_at, - UTC_TIMESTAMP() AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, COUNT(*) AS count, SUM(CASE WHEN `delayed_jobs`.`run_at` > '2025-11-10 17:20:13' THEN 1 ELSE 0 END) AS future_count, - SUM(CASE WHEN `delayed_jobs`.`attempts` > 0 THEN 1 ELSE 0 END) AS erroring_count, - MIN(CASE WHEN `delayed_jobs`.`run_at` <= '2025-11-10 17:20:13' THEN run_at ELSE NULL END) AS run_at + SUM(CASE WHEN `delayed_jobs`.`attempts` > 0 THEN 1 ELSE 0 END) AS erroring_count FROM `delayed_jobs` WHERE `delayed_jobs`.`failed_at` IS NULL GROUP BY `delayed_jobs`.`priority`, `delayed_jobs`.`queue`) subquery @@ -479,13 +476,16 @@ SELECT SUM(count) AS count, SELECT SUM(claimed_count) AS claimed_count, SUM(claimable_count) AS claimable_count, MIN(locked_at) AS locked_at, + MIN(run_at) AS run_at, UTC_TIMESTAMP() AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, SUM(CASE WHEN `delayed_jobs`.`locked_at` >= '2025-11-10 16:59:43' THEN 1 ELSE 0 END) AS claimed_count, SUM(CASE WHEN (`delayed_jobs`.`locked_at` IS NULL OR `delayed_jobs`.`locked_at` < '2025-11-10 16:59:43') THEN 1 ELSE 0 END) AS claimable_count, - MIN(CASE WHEN `delayed_jobs`.`locked_at` >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at + MIN(CASE WHEN `delayed_jobs`.`locked_at` >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at, + MIN(CASE WHEN (`delayed_jobs`.`locked_at` IS NULL + OR `delayed_jobs`.`locked_at` < '2025-11-10 16:59:43') THEN run_at ELSE NULL END) AS run_at FROM `delayed_jobs` WHERE `delayed_jobs`.`failed_at` IS NULL AND `delayed_jobs`.`run_at` <= '2025-11-10 17:20:13' @@ -548,14 +548,11 @@ SELECT SUM(count) AS count, SELECT SUM(count) AS count, SUM(future_count) AS future_count, SUM(erroring_count) AS erroring_count, - MIN(run_at) AS run_at, - UTC_TIMESTAMP() AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, COUNT(*) AS count, SUM(CASE WHEN `delayed_jobs`.`run_at` > '2025-11-10 17:20:13' THEN 1 ELSE 0 END) AS future_count, - SUM(CASE WHEN `delayed_jobs`.`attempts` > 0 THEN 1 ELSE 0 END) AS erroring_count, - MIN(CASE WHEN `delayed_jobs`.`run_at` <= '2025-11-10 17:20:13' THEN run_at ELSE NULL END) AS run_at + SUM(CASE WHEN `delayed_jobs`.`attempts` > 0 THEN 1 ELSE 0 END) AS erroring_count FROM `delayed_jobs` WHERE `delayed_jobs`.`failed_at` IS NULL GROUP BY `delayed_jobs`.`priority`, `delayed_jobs`.`queue`) subquery @@ -578,13 +575,16 @@ SELECT SUM(count) AS count, SELECT SUM(claimed_count) AS claimed_count, SUM(claimable_count) AS claimable_count, MIN(locked_at) AS locked_at, + MIN(run_at) AS run_at, UTC_TIMESTAMP() AS db_now_utc, CASE WHEN priority < 10 THEN 0 WHEN priority < 20 THEN 10 WHEN priority < 30 THEN 20 WHEN priority >= 30 THEN 30 END AS priority, queue AS queue FROM (SELECT `delayed_jobs`.`priority`, `delayed_jobs`.`queue`, SUM(CASE WHEN `delayed_jobs`.`locked_at` >= '2025-11-10 16:59:43' THEN 1 ELSE 0 END) AS claimed_count, SUM(CASE WHEN (`delayed_jobs`.`locked_at` IS NULL OR `delayed_jobs`.`locked_at` < '2025-11-10 16:59:43') THEN 1 ELSE 0 END) AS claimable_count, - MIN(CASE WHEN `delayed_jobs`.`locked_at` >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at + MIN(CASE WHEN `delayed_jobs`.`locked_at` >= '2025-11-10 16:59:43' THEN locked_at ELSE NULL END) AS locked_at, + MIN(CASE WHEN (`delayed_jobs`.`locked_at` IS NULL + OR `delayed_jobs`.`locked_at` < '2025-11-10 16:59:43') THEN run_at ELSE NULL END) AS run_at FROM `delayed_jobs` WHERE `delayed_jobs`.`failed_at` IS NULL AND `delayed_jobs`.`run_at` <= '2025-11-10 17:20:13' diff --git a/spec/delayed/monitor_spec.rb b/spec/delayed/monitor_spec.rb index 2e16169..4e13fa9 100644 --- a/spec/delayed/monitor_spec.rb +++ b/spec/delayed/monitor_spec.rb @@ -344,6 +344,44 @@ end end end + + context 'when a job is locked (in-flight)' do + let(:payload) { default_payload.merge(priority: 'interactive') } + let(:base_attributes) do + { + priority: 0, + queue: 'default', + handler: "--- !ruby/object:SimpleJob\n", + name: 'SimpleJob', + attempts: 0, + } + end + + # A single slow, in-flight job: old run_at, but currently locked by a worker. + let!(:locked_job) { Delayed::Job.create! base_attributes.merge(run_at: now - 1.hour, locked_at: now - 5.minutes) } + + it 'excludes the locked job from max_age and alert_age_percent' do + expect { subject.run! } + .to emit_notification("delayed.job.locked_count").with_payload(payload).with_value(1) + .and emit_notification("delayed.job.workable_count").with_payload(payload).with_value(0) + .and emit_notification("delayed.job.max_lock_age").with_payload(payload).approximately.with_value(5.minutes) + .and emit_notification("delayed.job.max_age").with_payload(payload).approximately.with_value(0) + .and emit_notification("delayed.job.alert_age_percent").with_payload(payload).approximately.with_value(0) + end + + context 'and a workable job is also present in the same group' do + # The workable job's run_at is newer than the locked job's, so max_age must + # track the workable job (30s), not the locked job (1 hour). + let!(:workable_job) { Delayed::Job.create! base_attributes.merge(run_at: now - 30.seconds) } + + it 'reports max_age from the workable job only' do + expect { subject.run! } + .to emit_notification("delayed.job.locked_count").with_payload(payload).with_value(1) + .and emit_notification("delayed.job.workable_count").with_payload(payload).with_value(1) + .and emit_notification("delayed.job.max_age").with_payload(payload).approximately.with_value(30.seconds) + end + end + end end describe 'SQL' do From 0cbba3ade7fbb3082ec557fc05b951c5a2f945cf Mon Sep 17 00:00:00 2001 From: Harris Effron Date: Mon, 15 Jun 2026 10:32:21 -0400 Subject: [PATCH 2/2] bump version --- Gemfile.lock | 2 +- lib/delayed/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 6be79ad..e99178d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - delayed (3.0.0) + delayed (3.0.1) activerecord (>= 6.0) concurrent-ruby diff --git a/lib/delayed/version.rb b/lib/delayed/version.rb index 717020b..6c5ef3a 100644 --- a/lib/delayed/version.rb +++ b/lib/delayed/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module Delayed - VERSION = '3.0.0' + VERSION = '3.0.1' end