feat(deps): use html2rss in latest development status

Signed-off-by: Gil Desmarais <git@desmarais.de>

Author: gildesmarais

Gemfile

@@ -4,7 +4,9 @@ source 'https://rubygems.org'
 
 git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
 
-gem 'html2rss', '~> 0.14'
+# gem 'html2rss', '~> 0.14'
+gem 'html2rss', github: 'html2rss/html2rss'
+
 gem 'html2rss-configs', github: 'html2rss/html2rss-configs'
 
 # Use these instead of the two above (uncomment them) when developing locally:

Gemfile.lock

@@ -84,11 +84,38 @@ def self.development? = ENV['RACK_ENV'] == 'development'
         end
 
         r.on String, String do |folder_name, config_name_with_ext|
-          handle_html2rss_configs(request, folder_name, config_name_with_ext)
+          response['Content-Type'] = CONTENT_TYPE_RSS
+
+          name = "#{folder_name}/#{File.basename(config_name_with_ext, '.*')}"
+          config = Html2rss::Configs.find_by_name(name)
+
+          if (params = request.params).any?
+            config[:params] ||= {}
+            config[:params].merge!(params)
+          end
+
+          feed = Html2rss.feed(config)
+
+          HttpCache.expires(response, feed.channel.ttl.to_i * 60, cache_control: 'public')
+
+          feed.to_xml
         end
 
         r.on String do |config_name_with_ext|
-          handle_local_config_feeds(request, config_name_with_ext)
+          response['Content-Type'] = CONTENT_TYPE_RSS
+
+          config = LocalConfig.find(File.basename(config_name_with_ext, '.*'))
+
+          if (params = request.params).any?
+            config[:params] ||= {}
+            config[:params].merge!(params)
+          end
+
+          feed = Html2rss.feed(config)
+
+          HttpCache.expires(response, feed.channel.ttl.to_i * 60, cache_control: 'public')
+
+          feed.to_xml
         end
       end
 

app.rb

@@ -14,34 +14,47 @@ class Html2rssFacade
 
       attr_reader :feed_config, :typecast_params
 
-      ##
-      # @param name [String] the name of a html2rss-configs provided config.
-      # @param typecast_params [Object]
-      # @return [String] the serialized RSS feed
-      def self.from_config(name, typecast_params, &)
-        feed_config = Html2rss::Configs.find_by_name(name)
-        new(feed_config, typecast_params).feed(&)
-      end
+      class << self
+        ##
+        # @param name [String] the name of a html2rss-configs provided config.
+        # @param typecast_params [Object]
+        # @return [String] the serialized RSS feed
+        def from_config(name, typecast_params, &)
+          feed_config = Html2rss::Configs.find_by_name(name)
+          new(feed_config, typecast_params).feed(&)
+        end
 
-      ##
-      # @param name [String] the name of a feed in the file `config/feeds.yml`
-      # @param typecast_params [Object]
-      # @return [String] the serialized RSS feed
-      def self.from_local_config(name, typecast_params, &)
-        feed_config = LocalConfig.find(name)
-        new(feed_config, typecast_params).feed(&)
-      end
+        ##
+        # @param name [String] the name of a feed in the file `config/feeds.yml`
+        # @param typecast_params [Object]
+        # @return [String] the serialized RSS feed
+        def from_local_config(name, typecast_params, &)
+          feed_config = LocalConfig.find(name)
+          new(feed_config, typecast_params).feed(&)
+        end
 
-      ##
-      # @param feed_config [Hash<Symbol, Object>]
-      # @param typecast_params [Object]
-      # @param global_config [Hash<Symbol, Object>]
-      # @return [Html2rss::Config]
-      # @raise [Roda::RodaPlugins::TypecastParams::Error]
-      def self.feed_config_to_config(feed_config, typecast_params, global_config: LocalConfig.global)
-        dynamic_params = Html2rss::Config::Channel.required_params_for_config(feed_config[:channel])
-                                                  .to_h { |name| [name, typecast_params.str!(name)] }
-        Html2rss::Config.new(feed_config, global_config, dynamic_params)
+        ##
+        # @param feed_config [Hash<Symbol, Object>]
+        # @param typecast_params [Object]
+        # @param global_config [Hash<Symbol, Object>]
+        # @return [Html2rss::Config]
+        # @raise [Roda::RodaPlugins::TypecastParams::Error]
+        def feed_config_to_config(feed_config, typecast_params, global_config: LocalConfig.global)
+          params = required_params_for_config(feed_config[:channel])
+                   .to_h { |name| [name, typecast_params.str!(name)] }
+
+          config = global_config.merge(feed_config)
+          config[:params] = params if params.any?
+          config
+        end
+
+        private
+
+        def required_params_for_config(config)
+          config.each_with_object(Set.new) do |(_, value), required_params|
+            required_params.merge(value.scan(/%<(\w+)>[s|d]/).flatten) if value.is_a?(String)
+          end
+        end
       end
 
       ##

app/html2rss_facade.rb

@@ -15,6 +15,8 @@ module HttpCache
       # @param seconds [Integer]
       # @param cache_control [String, nil]
       def expires(response, seconds, cache_control: nil)
+        expires_now(response) and return if seconds <= 0
+
         response['Expires'] = (Time.now + seconds).httpdate
 
         cache_value = "max-age=#{seconds}"

app/http_cache.rb

@@ -16,6 +16,7 @@ def execute
         response = SsrfFilter.get(ctx.url, headers:)
 
         Html2rss::RequestService::Response.new(body: response.body,
+                                               url: ctx.url,
                                                headers: response.to_hash.transform_values(&:first))
       end
     end

app/ssrf_filter_strategy.rb

@@ -16,6 +16,6 @@ feeds:
         selector: "li > div"
       title:
         selector: "h4"
-      link:
+      url:
         selector: "a"
         extractor: "href"

config/feeds.yml

@@ -8,12 +8,11 @@ module Web
     class App
       def handle_error(error) # rubocop:disable Metrics/MethodLength
         case error
-        when Html2rss::Config::ParamsMissing,
+        when Html2rss::Config::DynamicParams::ParamsMissing,
              Roda::RodaPlugins::TypecastParams::Error
           set_error_response('Parameters missing or invalid', 422)
-        when Html2rss::AttributePostProcessors::UnknownPostProcessorName,
-             Html2rss::ItemExtractors::UnknownExtractorName,
-             Html2rss::Config::ChannelMissing
+        when Html2rss::Selectors::PostProcessors::UnknownPostProcessorName,
+             Html2rss::Selectors::Extractors::UnknownExtractorName
           set_error_response('Invalid feed config', 422)
         when LocalConfig::NotFound,
              Html2rss::Configs::ConfigNotFound

helpers/handle_error.rb

@@ -8,7 +8,8 @@ def handle_html2rss_configs(request, _folder_name, _config_name_with_ext)
 
         Html2rssFacade.from_config(path.full_config_name, typecast_params) do |config|
           response['Content-Type'] = CONTENT_TYPE_RSS
-          HttpCache.expires(response, config.ttl * 60, cache_control: 'public')
+
+          HttpCache.expires(response, config.dig(:channel, :ttl).to_i * 60, cache_control: 'public')
         end
       end
     end

helpers/handle_html2rss_configs.rb

@@ -8,7 +8,8 @@ def handle_local_config_feeds(request, _config_name_with_ext)
 
         Html2rssFacade.from_local_config(path.full_config_name, typecast_params) do |config|
           response['Content-Type'] = CONTENT_TYPE_RSS
-          HttpCache.expires(response, config.ttl * 60, cache_control: 'public')
+
+          HttpCache.expires(response, config.dig(:channel, :ttl).to_i * 60, cache_control: 'public')
         end
       end
     end