feat(config): add E1 browserless configs and fetch lane

Author: gildesmarais

Makefile

@@ -19,7 +19,10 @@ test-fetch-changed-configs:
 	bin/rspec_changed_configs
 
 test-fetch-all-configs:
-	bundle exec rspec --tag fetch spec/html2rss/configs
+	bundle exec rspec --tag fetch spec/html2rss/configs_dynamic_spec.rb
+
+test-fetch-browserless-configs:
+	bin/rspec_browserless_configs
 
 test-all: test test-fetch-all-configs
 

README.md

@@ -76,10 +76,23 @@ make test-config CONFIG=github.com/releases.yml
 
 # Test domain
 make test-domain DOMAIN=github.com
+
+# Run live fetch tests for the full corpus
+make test-fetch-all-configs
+
+# Run the Browserless-backed fetch subset
+BROWSERLESS_IO_WEBSOCKET_URL=ws://127.0.0.1:4002 \
+BROWSERLESS_IO_API_TOKEN=... \
+make test-fetch-browserless-configs
 ```
 
 **Adding new configs**: Create the YAML file, run `make validate`, then run the generated tests. No dedicated spec file is needed.
 
+The fetch suite has two lanes:
+
+- `make test-fetch-all-configs` runs all `:fetch` examples. Configs marked as Browserless-backed are skipped unless Browserless env vars are configured.
+- `make test-fetch-browserless-configs` runs only the Browserless-backed config subset and requires `BROWSERLESS_IO_WEBSOCKET_URL`. Custom endpoints also require `BROWSERLESS_IO_API_TOKEN`.
+
 **Config folder convention**: Place configs under the registrable domain folder (e.g., `example.com/` or `bbc.co.uk/`). Legacy subdomain folders (e.g., `news.example.com/`) are allowed but not preferred.
 
 ## Editor Setup (JSON Schema)

bin/rspec_browserless_configs

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../spec/support/browserless_fetch_configs'
+
+unless BrowserlessFetchConfigs.browserless_env_configured?
+  warn 'BROWSERLESS_IO_WEBSOCKET_URL is required for browserless fetch tests.'
+  warn 'Set BROWSERLESS_IO_API_TOKEN as well when using a custom websocket endpoint.'
+  exit 1
+end
+
+args = ['bundle', 'exec', 'rspec', '--tag', 'fetch']
+BrowserlessFetchConfigs::CONFIGS.each do |config|
+  args << '--example'
+  args << config
+end
+args << 'spec/html2rss/configs_dynamic_spec.rb'
+
+exec(*args)

lib/html2rss/configs/apple.com/newsroom.yml

@@ -0,0 +1,18 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json
+strategy: browserless
+
+channel:
+  url: https://www.apple.com/newsroom/
+  language: en
+  time_zone: UTC
+  ttl: 360
+selectors:
+  items:
+    selector: 'li.tile-item a.tile-hero, li.tile-item a.tile-2up, li.tile-item a.tile-3up, li.tile-item a.tile-list'
+    enhance: false
+  title:
+    selector: .tile__headline
+  url:
+    extractor: href
+  published_at:
+    selector: .tile__timestamp

lib/html2rss/configs/deepmind.google/blog.yml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json
+strategy: browserless
+
+channel:
+  url: https://deepmind.google/blog/
+  language: en
+  time_zone: UTC
+  ttl: 360
+selectors:
+  items:
+    selector: .card__inner
+    enhance: false
+  title:
+    selector: h3
+  url:
+    selector: .card__overlay-link
+    extractor: href
+  published_at:
+    selector: time

lib/html2rss/configs/notion.com/blog.yml

@@ -0,0 +1,19 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json
+strategy: browserless
+
+channel:
+  url: https://www.notion.com/blog
+  language: en
+  time_zone: UTC
+  ttl: 360
+selectors:
+  items:
+    selector: article.post-preview
+    enhance: false
+  title:
+    selector: h3 a[href*="/blog/"]
+  url:
+    selector: h3 a[href*="/blog/"]
+    extractor: href
+  description:
+    selector: '> a[href*="/blog/"]:not([title])'

lib/html2rss/configs/shopify.com/blog.yml

@@ -0,0 +1,16 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json
+strategy: browserless
+
+channel:
+  url: https://www.shopify.com/blog/latest
+  time_zone: UTC
+  ttl: 360
+selectors:
+  items:
+    selector: article.article--index
+    enhance: false
+  title:
+    selector: '.blogPost a[href*="/blog/"]:not([href*="/topics/"])'
+  url:
+    selector: '.blogPost a[href*="/blog/"]:not([href*="/topics/"])'
+    extractor: href

lib/html2rss/configs/spotify.com/newsroom.yml

@@ -0,0 +1,17 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/html2rss/html2rss/refs/heads/master/schema/html2rss-config.schema.json
+strategy: browserless
+
+channel:
+  url: https://newsroom.spotify.com/
+  language: en
+  time_zone: UTC
+  ttl: 360
+selectors:
+  items:
+    selector: '.post-box.v2'
+    enhance: false
+  title:
+    selector: 'h3 a[href*="/20"]'
+  url:
+    selector: 'h3 a[href*="/20"]'
+    extractor: href

spec/browserless_fetch_configs_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+RSpec.describe BrowserlessFetchConfigs do
+  describe '.browserless_env_configured?' do
+    around do |example|
+      original_ws_url = ENV['BROWSERLESS_IO_WEBSOCKET_URL']
+      original_api_token = ENV['BROWSERLESS_IO_API_TOKEN']
+
+      example.run
+    ensure
+      ENV['BROWSERLESS_IO_WEBSOCKET_URL'] = original_ws_url
+      ENV['BROWSERLESS_IO_API_TOKEN'] = original_api_token
+    end
+
+    it 'accepts the documented local Browserless websocket URL without a token' do
+      ENV['BROWSERLESS_IO_WEBSOCKET_URL'] = 'ws://127.0.0.1:4002'
+      ENV['BROWSERLESS_IO_API_TOKEN'] = ''
+
+      expect(described_class.browserless_env_configured?).to be(true)
+    end
+
+    it 'accepts the legacy local Browserless websocket URL without a token' do
+      ENV['BROWSERLESS_IO_WEBSOCKET_URL'] = 'ws://127.0.0.1:3000'
+      ENV['BROWSERLESS_IO_API_TOKEN'] = ''
+
+      expect(described_class.browserless_env_configured?).to be(true)
+    end
+
+    it 'requires a token for non-local websocket URLs' do
+      ENV['BROWSERLESS_IO_WEBSOCKET_URL'] = 'wss://production.browserless.example/ws'
+      ENV['BROWSERLESS_IO_API_TOKEN'] = ''
+
+      expect(described_class.browserless_env_configured?).to be(false)
+    end
+  end
+end

spec/support/browserless_fetch_configs.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module BrowserlessFetchConfigs
+  LOCAL_WS_URLS = %w[
+    ws://127.0.0.1:3000
+    ws://127.0.0.1:4002
+  ].freeze
+
+  CONFIGS = %w[
+    apple.com/newsroom.yml
+    deepmind.google/blog.yml
+    notion.com/blog.yml
+    shopify.com/blog.yml
+    spotify.com/newsroom.yml
+  ].freeze
+
+  module_function
+
+  def include?(file_name)
+    CONFIGS.include?(file_name)
+  end
+
+  def browserless_env_configured?
+    ws_url = ENV['BROWSERLESS_IO_WEBSOCKET_URL'].to_s
+    return false if ws_url.empty?
+    return true if LOCAL_WS_URLS.include?(ws_url)
+
+    !ENV['BROWSERLESS_IO_API_TOKEN'].to_s.empty?
+  end
+end