Add DSN-gated Sentry breadcrumbs and triage runbook

Author: gildesmarais

app/web/boot/sentry.rb

@@ -28,6 +28,7 @@ def initialize_sentry!
             ::Sentry.init do |config|
               apply_settings(config)
             end
+            apply_scope_tags!
           end
 
           # @param config [Object]
@@ -45,6 +46,20 @@ def release_name
             "#{RuntimeEnv.build_tag}+#{RuntimeEnv.git_sha}"
           end
 
+          # @return [void]
+          def apply_scope_tags!
+            return unless defined?(::Sentry) && ::Sentry.respond_to?(:configure_scope)
+
+            ::Sentry.configure_scope do |scope|
+              scope.set_tags(
+                release: release_name,
+                environment: RuntimeEnv.rack_env
+              )
+            end
+          rescue StandardError
+            nil
+          end
+
           # @return [Boolean]
           def sentry_initialized?
             defined?(::Sentry) && ::Sentry.respond_to?(:initialized?) && ::Sentry.initialized?

app/web/telemetry/app_logger.rb

@@ -3,11 +3,8 @@
 require 'json'
 require 'logger'
 require 'time'
-
 module Html2rss
   module Web
-    ##
-    # Shared structured logger for application and middleware runtime events.
     module AppLogger
       class << self
         # @return [Logger]
@@ -103,6 +100,7 @@ def normalize_logfmt_value(raw_value)
         def emit_to_sentry(payload)
           return unless sentry_payload?(payload)
 
+          SentryLogs.record_breadcrumb(payload)
           SentryLogs.emit(payload)
         rescue StandardError
           nil

app/web/telemetry/sentry_logs.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative '../security/log_sanitizer'
+
 module Html2rss
   module Web
     ##
@@ -9,8 +11,26 @@ module SentryLogs
       OMIT = Object.new.freeze
       ALLOWED_LEVELS = %i[debug info warn error fatal].freeze
       SENSITIVE_ATTRIBUTE_KEYS = %w[actor email ip remote_ip user_agent username x_forwarded_for].freeze
+      BREADCRUMB_KEYS = %i[event_name security_event outcome request_id route_group strategy component details].freeze
+      BREADCRUMB_CATEGORY_KEYS = %i[event_name security_event component].freeze
+      BREADCRUMB_MESSAGE_KEYS = %i[message event_name security_event component].freeze
 
       class << self
+        # @param payload [Hash{Symbol=>Object}]
+        # @return [void]
+        def record_breadcrumb(payload)
+          return unless breadcrumb_enabled?
+
+          ::Sentry.add_breadcrumb(
+            category: breadcrumb_category(payload),
+            message: breadcrumb_message(payload),
+            level: breadcrumb_level(payload),
+            data: breadcrumb_data(payload)
+          )
+        rescue StandardError
+          nil
+        end
+
         # @param payload [Hash{Symbol=>Object}]
         # @return [void]
         def emit(payload)
@@ -31,6 +51,13 @@ def enabled?
             !logger.nil?
         end
 
+        # @return [Boolean]
+        def breadcrumb_enabled?
+          RuntimeEnv.sentry_enabled? &&
+            defined?(::Sentry) &&
+            ::Sentry.respond_to?(:add_breadcrumb)
+        end
+
         # @return [Object, nil]
         def logger
           return unless defined?(::Sentry) && ::Sentry.respond_to?(:logger)
@@ -54,6 +81,34 @@ def message(payload)
             payload[:component] || 'html2rss-web log'
         end
 
+        # @param payload [Hash{Symbol=>Object}]
+        # @return [String]
+        def breadcrumb_category(payload)
+          breadcrumb_label(payload, 'html2rss-web', BREADCRUMB_CATEGORY_KEYS)
+        end
+
+        # @param payload [Hash{Symbol=>Object}]
+        # @return [String]
+        def breadcrumb_message(payload)
+          breadcrumb_label(payload, 'html2rss-web log', BREADCRUMB_MESSAGE_KEYS)
+        end
+
+        # @param payload [Hash{Symbol=>Object}]
+        # @return [String]
+        def breadcrumb_level(payload)
+          requested_level = payload.fetch(:level, 'info').to_s.downcase
+          return 'warning' if requested_level == 'warn'
+          return requested_level if ALLOWED_LEVELS.map(&:to_s).include?(requested_level)
+
+          'info'
+        end
+
+        # @param payload [Hash{Symbol=>Object}]
+        # @return [Hash{Symbol=>Object}]
+        def breadcrumb_data(payload)
+          LogSanitizer.sanitize_details(payload).slice(*BREADCRUMB_KEYS)
+        end
+
         # @param payload [Hash{Symbol=>Object}]
         # @return [Hash{Symbol=>Object}]
         def attributes(payload)
@@ -98,6 +153,14 @@ def sanitize_array(key, values)
         def sensitive_key?(key)
           SENSITIVE_ATTRIBUTE_KEYS.include?(key.to_s)
         end
+
+        # @param payload [Hash{Symbol=>Object}]
+        # @param fallback [String]
+        # @param keys [Array<Symbol>]
+        # @return [String]
+        def breadcrumb_label(payload, fallback, keys)
+          keys.lazy.map { |key| payload[key] }.find(&:itself) || fallback
+        end
       end
     end
   end

docs/README.md

@@ -197,6 +197,25 @@ Canonical event fields: `event_name`, `schema_version`, `request_id`, `route_gro
 
 Critical-path event families: auth, feed create, feed render, request errors.
 
+## Sentry Runbook
+
+When `SENTRY_DSN` is present, Sentry is enabled. `BUILD_TAG` and `GIT_SHA` become the release identifier, and
+`RACK_ENV` becomes the environment tag.
+
+First-15-minute triage checklist:
+
+1. Open the newest `feed.create`, `feed.render`, and `request.error` events.
+2. Confirm the release tag matches the deployed build.
+3. Check the breadcrumb trail for the failing path, strategy, and outcome.
+4. Decide whether the failure is retryable or terminal before paging the user-facing incident path.
+
+Alert baseline:
+
+1. Page on sustained `request.error` spikes in production.
+2. Page on a repeated `feed.render` failure burst, especially when success drops to zero for a route or strategy.
+3. Track the first recovery signal after fallback or retry succeeds so the incident can be closed quickly.
+4. Keep the initial threshold simple; tune after a few real incidents instead of pre-optimizing for every edge case.
+
 ---
 
 ## Documentation Policy

spec/html2rss/web/boot/sentry_spec.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require_relative '../../../../app'
+
+RSpec.describe Html2rss::Web::Boot::Sentry do
+  let(:sentry_dsn) { 'https://example@sentry.invalid/1' }
+  let(:captured_config) { Struct.new(:dsn, :environment, :enable_logs, :send_default_pii, :release).new }
+  let(:captured_scope) do
+    Class.new do
+      attr_reader :tags
+
+      def initialize
+        @tags = {}
+      end
+
+      def set_tags(**tags)
+        @tags = tags
+      end
+    end.new
+  end
+  let(:fake_sentry) do
+    config = captured_config
+    scope = captured_scope
+
+    Module.new.tap do |mod|
+      mod.define_singleton_method(:initialized?) { false }
+      mod.define_singleton_method(:init) do |&block|
+        block.call(config)
+      end
+      mod.define_singleton_method(:configure_scope) do |&block|
+        block.call(scope)
+      end
+    end
+  end
+
+  before do
+    stub_const('Sentry', fake_sentry)
+  end
+
+  it 'configures release, environment, and scope tags when a dsn is present', :aggregate_failures do
+    stub_runtime_env_for_sentry('production')
+    described_class.send(:initialize_sentry!)
+
+    expect_sentry_configuration
+  end
+
+  it 'does nothing when a dsn is not present' do
+    allow(Html2rss::Web::RuntimeEnv).to receive(:sentry_enabled?).and_return(false)
+
+    expect(described_class.send(:configure?)).to be(false)
+  end
+
+  def stub_runtime_env_for_sentry(rack_env)
+    allow(Html2rss::Web::RuntimeEnv).to receive_messages(
+      sentry_enabled?: true,
+      sentry_dsn: sentry_dsn,
+      rack_env: rack_env,
+      build_tag: '2026-03-27',
+      git_sha: 'abc1234',
+      sentry_logs_enabled?: false
+    )
+  end
+
+  def expect_sentry_configuration
+    expect(captured_config).to have_attributes(
+      dsn: sentry_dsn,
+      environment: 'production',
+      enable_logs: false,
+      send_default_pii: false,
+      release: '2026-03-27+abc1234'
+    )
+    expect_sentry_scope_tags
+  end
+
+  def expect_sentry_scope_tags
+    expect(captured_scope.tags).to eq(
+      release: '2026-03-27+abc1234',
+      environment: 'production'
+    )
+  end
+end

spec/html2rss/web/sentry_logs_spec.rb

@@ -3,6 +3,7 @@
 require 'spec_helper'
 
 require_relative '../../../app/web/config/runtime_env'
+require_relative '../../../app/web/telemetry/app_logger'
 require_relative '../../../app/web/telemetry/sentry_logs'
 
 RSpec.describe Html2rss::Web::SentryLogs do
@@ -23,6 +24,7 @@
   let(:fake_sentry) do
     Module.new.tap do |mod|
       mod.define_singleton_method(:logger) { sentry_logger }
+      mod.define_singleton_method(:add_breadcrumb) { |**| nil }
     end
   end
   let(:raw_payload) do
@@ -65,6 +67,22 @@
     expect(captured_call).to eq({})
   end
 
+  it 'adds breadcrumbs for request-critical structured logs even when sentry logs are disabled', :aggregate_failures do
+    stub_const('Sentry', fake_sentry)
+    allow(Html2rss::Web::RuntimeEnv).to receive_messages(sentry_enabled?: true, sentry_logs_enabled?: false)
+    allow(Sentry).to receive(:add_breadcrumb)
+
+    Html2rss::Web::AppLogger.send(
+      :format_entry,
+      'INFO',
+      Time.now.utc,
+      nil,
+      breadcrumb_payload.to_json
+    )
+
+    expect(Sentry).to have_received(:add_breadcrumb).with(expected_breadcrumb)
+  end
+
   it 'falls back to info when an unsupported level is requested', :aggregate_failures do
     stub_const('Sentry', fake_sentry)
     allow(Html2rss::Web::RuntimeEnv).to receive_messages(sentry_enabled?: true, sentry_logs_enabled?: true)
@@ -80,6 +98,44 @@ def build_sentry_logger
     logger_class.new(captured_call)
   end
 
+  def breadcrumb_payload
+    {
+      event_name: 'feed.create',
+      outcome: 'failure',
+      request_id: 'req-123',
+      route_group: 'api_v1',
+      strategy: 'faraday',
+      details: { url: 'https://example.com/articles', fallback: 'browserless' }
+    }
+  end
+
+  def expected_breadcrumb
+    include(
+      category: 'feed.create',
+      message: 'feed.create',
+      level: 'info',
+      data: breadcrumb_data_matcher
+    )
+  end
+
+  def breadcrumb_data_matcher
+    include(
+      event_name: 'feed.create',
+      outcome: 'failure',
+      request_id: 'req-123',
+      route_group: 'api_v1',
+      strategy: 'faraday',
+      details: breadcrumb_details_matcher
+    )
+  end
+
+  def breadcrumb_details_matcher
+    include(
+      url: include(host: 'example.com', scheme: 'https'),
+      fallback: 'browserless'
+    )
+  end
+
   def expect_forwarded_payload
     expect(captured_call).to include(:message, :attributes)
     expect_forwarded_message