feat: introduce pluggable HTTP client behaviour

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## Unreleased
+
+- Add pluggable HTTP client behaviour (`Waffle.HTTPClient`) with `Waffle.HTTPClient.Hackney`
+  as the default implementation (#150)
+  - `:timeout` and `:recv_timeout` error atoms are **unchanged** from previous behaviour
+  - `{:error, :service_unavailable}` replaces `{:error, {:waffle_hackney_error, {:ok, 503, ...}}}` on 503 out of retries
+  - Non-2xx errors now return `{:error, {:http_error, status_code}}` (e.g. `{:error, {:http_error, 404}}`), replacing `{:error, {:waffle_hackney_error, {:ok, status, headers, ref}}}`
+
 ## v1.1.10 (2025-12-28)
 
 - Handle http errors from fetching remote files (#134)

README.md

@@ -80,6 +80,16 @@ config :ex_aws,
   # any configurations provided by https://github.com/ex-aws/ex_aws
 ```
 
+### Setup an HTTP client
+
+Waffle uses `:hackney` by default to download remote files. You can configure it explicitly:
+
+```elixir
+config :waffle, :http_client, Waffle.HTTPClient.Hackney
+```
+
+You can also implement your own client by adopting the `Waffle.HTTPClient` behaviour.
+
 ### Define a definition module
 
 Waffle requires a **definition module** which contains the relevant

lib/waffle/behaviors/http_client_behaviour.ex

@@ -0,0 +1,55 @@
+defmodule Waffle.HTTPClient do
+  @moduledoc """
+  Behaviour for pluggable HTTP clients used when downloading remote files.
+
+  ## Built-in implementations
+
+  - `Waffle.HTTPClient.Hackney` — default, uses `:hackney`. Add `{:hackney, "~> 1.9"}` to
+    your deps.
+
+  ## Configuration
+
+      config :waffle, :http_client, Waffle.HTTPClient.Hackney
+
+  ## Writing a custom client
+
+  Implement the `c:get/3` callback and configure Waffle to use your module:
+
+      config :waffle, :http_client, MyApp.HTTPClient
+
+  ### Options
+
+  Waffle passes the following options (all values come from application config):
+
+  | Option             | Type                    | Default    | Description                              |
+  |--------------------|-------------------------|------------|------------------------------------------|
+  | `:recv_timeout`    | `non_neg_integer()`              | `5_000`     | Timeout for receiving a response (ms)              |
+  | `:connect_timeout` | `non_neg_integer()`              | `10_000`    | Timeout for establishing a connection (ms)         |
+  | `:max_body_length` | `non_neg_integer() \| :infinity` | `:infinity` | Maximum allowed response body size (bytes)         |
+  | `:follow_redirect` | `boolean()`                      | `true`      | Whether to follow HTTP redirects                   |
+
+  ### Return values
+
+  | Pattern                                | Meaning                                          |
+  |----------------------------------------|--------------------------------------------------|
+  | `{:ok, body}`                          | Successful response, no filename in headers      |
+  | `{:ok, body, filename}`                | Successful response with `content-disposition` filename |
+  | `{:error, :timeout}`                   | Connect timed out — Waffle will retry            |
+  | `{:error, :recv_timeout}`              | Receive timed out — Waffle will retry            |
+  | `{:error, :service_unavailable}`       | Server returned 503 — Waffle will retry          |
+  | `{:error, {:http_error, reason}}`      | Non-retryable error; `reason` is the HTTP status integer for unexpected status codes, or an error term for connection/protocol errors |
+  """
+
+  @type body :: binary()
+  @type filename :: String.t()
+  @type option ::
+          {:recv_timeout, non_neg_integer()}
+          | {:connect_timeout, non_neg_integer()}
+          | {:max_body_length, non_neg_integer() | :infinity}
+          | {:follow_redirect, boolean()}
+
+  @callback get(url :: String.t(), headers :: list(), options :: [option()]) ::
+              {:ok, body()}
+              | {:ok, body(), filename()}
+              | {:error, :timeout | :recv_timeout | :service_unavailable | {:http_error, any()}}
+end

lib/waffle/file.ex

@@ -185,18 +185,14 @@ defmodule Waffle.File do
     end
   end
 
-  # hackney :connect_timeout - timeout used when establishing a connection, in milliseconds
-  # hackney :recv_timeout - timeout used when receiving from a connection, in milliseconds
-  # hackney :max_body_length - maximum size of the file to download, in bytes. Defaults to :infinity
-  # :backoff_max - maximum backoff time, in milliseconds
-  # :backoff_factor - a backoff factor to apply between attempts, in milliseconds
   defp get_remote_path(remote_path, definition) do
     headers = definition.remote_file_headers(remote_path)
 
     options = [
       follow_redirect: true,
       recv_timeout: Application.get_env(:waffle, :recv_timeout, 5_000),
       connect_timeout: Application.get_env(:waffle, :connect_timeout, 10_000),
+      max_body_length: Application.get_env(:waffle, :max_body_length, :infinity),
       max_retries: Application.get_env(:waffle, :max_retries, 3),
       backoff_factor: Application.get_env(:waffle, :backoff_factor, 1000),
       backoff_max: Application.get_env(:waffle, :backoff_max, 30_000)
@@ -206,80 +202,34 @@ defmodule Waffle.File do
   end
 
   defp request(remote_path, headers, options, tries \\ 0) do
-    with {:ok, 200, response_headers, client_ref} <-
-           :hackney.get(URI.to_string(remote_path), headers, "", options),
-         res when elem(res, 0) == :ok <- body(client_ref, response_headers) do
-      res
-    else
-      {:error, %{reason: :timeout}} ->
-        case retry(tries, options) do
-          {:ok, :retry} -> request(remote_path, headers, options, tries + 1)
-          {:error, :out_of_tries} -> {:error, :timeout}
-        end
-
-      {:error, :timeout} ->
-        case retry(tries, options) do
-          {:ok, :retry} -> request(remote_path, headers, options, tries + 1)
-          {:error, :out_of_tries} -> {:error, :recv_timeout}
-        end
-
-      {:ok, 503, _headers, client_ref} = response ->
-        case retry(tries, options) do
-          {:ok, :retry} ->
-            request(remote_path, headers, options, tries + 1)
-
-          {:error, :out_of_tries} ->
-            :hackney.close(client_ref)
-            {:error, {:waffle_hackney_error, response}}
-        end
-
-      {:ok, _, _, client_ref} = response ->
-        :hackney.close(client_ref)
-        {:error, {:waffle_hackney_error, response}}
-
-      _err ->
-        {:error, :waffle_hackney_error}
-    end
-  end
-
-  defp body(client_ref, response_headers) do
-    max_body_length = Application.get_env(:waffle, :max_body_length, :infinity)
+    http_client = Application.get_env(:waffle, :http_client, Waffle.HTTPClient.Hackney)
+    url = URI.to_string(remote_path)
 
-    case :hackney.body(client_ref, max_body_length) do
+    case http_client.get(url, headers, options) do
       {:ok, body} ->
-        response_headers = :hackney_headers.new(response_headers)
-        filename = content_disposition(response_headers)
+        {:ok, body}
 
-        if is_nil(filename) do
-          {:ok, body}
-        else
-          {:ok, body, filename}
-        end
+      {:ok, body, filename} ->
+        {:ok, body, filename}
 
-      err ->
+      {:error, reason} when reason in [:timeout, :recv_timeout, :service_unavailable] ->
+        retry_or_error(remote_path, headers, options, tries, reason)
+
+      {:error, _} = err ->
         err
     end
   end
 
-  defp content_disposition(headers) do
-    case :hackney_headers.get_value("content-disposition", headers) do
-      :undefined ->
-        nil
-
-      value ->
-        case :hackney_headers.content_disposition(value) do
-          {_, [{"filename", filename} | _]} ->
-            filename
-
-          _ ->
-            nil
-        end
+  defp retry_or_error(remote_path, headers, options, tries, reason) do
+    case retry(tries, options) do
+      {:ok, :retry} -> request(remote_path, headers, options, tries + 1)
+      {:error, :out_of_tries} -> {:error, reason}
     end
   end
 
   defp retry(tries, options) do
     if tries < options[:max_retries] do
-      backoff = round(options[:backoff_factor] * :math.pow(2, tries - 1))
+      backoff = round(options[:backoff_factor] * :math.pow(2, tries))
       backoff = :erlang.min(backoff, options[:backoff_max])
       :timer.sleep(backoff)
       {:ok, :retry}

lib/waffle/http_client/hackney.ex

@@ -0,0 +1,87 @@
+defmodule Waffle.HTTPClient.Hackney do
+  @moduledoc """
+  Default HTTP client implementation using `:hackney`.
+
+  Add `:hackney` to your dependencies:
+
+      {:hackney, "~> 1.9"}
+
+  ## Configuration
+
+      config :waffle, :http_client, Waffle.HTTPClient.Hackney
+
+  ## Options
+
+  | Option             | Default      | Description                                            |
+  |--------------------|--------------|--------------------------------------------------------|
+  | `:recv_timeout`    | `5_000`      | Timeout for receiving a response, in milliseconds      |
+  | `:connect_timeout` | `10_000`     | Timeout for establishing a connection, in milliseconds |
+  | `:max_body_length` | `:infinity`  | Maximum response body size, in bytes                   |
+  | `:follow_redirect` | `true`       | Whether to follow HTTP redirects automatically         |
+  """
+
+  @behaviour Waffle.HTTPClient
+
+  @impl Waffle.HTTPClient
+  def get(url, headers, options) do
+    hackney_options = [
+      follow_redirect: Keyword.get(options, :follow_redirect, true),
+      recv_timeout: Keyword.get(options, :recv_timeout, 5_000),
+      connect_timeout: Keyword.get(options, :connect_timeout, 10_000)
+    ]
+
+    max_body_length = Keyword.get(options, :max_body_length, :infinity)
+
+    case :hackney.get(url, headers, "", hackney_options) do
+      {:ok, 200, response_headers, client_ref} ->
+        read_body(client_ref, response_headers, max_body_length)
+
+      {:ok, 503, _headers, client_ref} ->
+        :hackney.close(client_ref)
+        {:error, :service_unavailable}
+
+      {:ok, status, _headers, client_ref} ->
+        :hackney.close(client_ref)
+        {:error, {:http_error, status}}
+
+      # connect timeout: hackney returns %{reason: :timeout}, not a bare atom
+      {:error, %{reason: :timeout}} ->
+        {:error, :timeout}
+
+      # recv timeout: hackney returns a bare :timeout atom
+      {:error, :timeout} ->
+        {:error, :recv_timeout}
+
+      {:error, reason} ->
+        {:error, {:http_error, reason}}
+    end
+  end
+
+  defp read_body(client_ref, response_headers, max_body_length) do
+    case :hackney.body(client_ref, max_body_length) do
+      {:ok, body} ->
+        filename =
+          :hackney_headers.new(response_headers)
+          |> get_content_disposition_filename()
+
+        if filename, do: {:ok, body, filename}, else: {:ok, body}
+
+      {:error, reason} ->
+        :hackney.close(client_ref)
+        {:error, {:http_error, reason}}
+    end
+  end
+
+  defp get_content_disposition_filename(headers) do
+    case :hackney_headers.get_value("content-disposition", headers) do
+      :undefined ->
+        nil
+
+      value ->
+        case :hackney_headers.content_disposition(value) do
+          {_, [{"filename", filename} | _]} -> filename
+          _ -> nil
+        end
+    end
+  end
+end

test/actions/store_test.exs

@@ -140,22 +140,34 @@ defmodule WaffleTest.Actions.Store do
   end
 
   test "recv_timeout" do
+    original = Application.get_env(:waffle, :recv_timeout)
     Application.put_env(:waffle, :recv_timeout, 1)
 
+    on_exit(fn ->
+      if original,
+        do: Application.put_env(:waffle, :recv_timeout, original),
+        else: Application.delete_env(:waffle, :recv_timeout)
+    end)
+
     with_mock Waffle.Storage.S3,
       put: fn DummyDefinition, _, {%{file_name: "favicon.ico", path: _}, nil} ->
         {:ok, "favicon.ico"}
       end do
       assert DummyDefinition.store("https://www.google.com/favicon.ico") ==
                {:error, :recv_timeout}
     end
-
-    Application.put_env(:waffle, :recv_timeout, 5_000)
   end
 
   test "recv_timeout with a filename" do
+    original = Application.get_env(:waffle, :recv_timeout)
     Application.put_env(:waffle, :recv_timeout, 1)
 
+    on_exit(fn ->
+      if original,
+        do: Application.put_env(:waffle, :recv_timeout, original),
+        else: Application.delete_env(:waffle, :recv_timeout)
+    end)
+
     with_mock Waffle.Storage.S3,
       put: fn DummyDefinition, _, {%{file_name: "newfavicon.ico", path: _}, nil} ->
         {:ok, "newfavicon.ico"}
@@ -166,8 +178,6 @@ defmodule WaffleTest.Actions.Store do
              }) ==
                {:error, :recv_timeout}
     end
-
-    Application.put_env(:waffle, :recv_timeout, 5_000)
   end
 
   test "accepts remote files" do