Skip to content

Commit 92a35e6

Browse files
louiscblouiscb
authored
Merge pull request #17 from AppMonet/louiscb/eng-2350-fallback-region-for-s3-configurable-in-plur-config
Add aws fallback region for rpt s3
2 parents 2fcc8be + ab57919 commit 92a35e6

3 files changed

Lines changed: 265 additions & 16 deletions

File tree

‎lib/remote_persistent_term/fetcher/s3.ex‎

Lines changed: 86 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
99
@type t :: %__MODULE__{
1010
bucket: String.t(),
1111
key: String.t(),
12-
region: String.t()
12+
region: String.t(),
13+
failover_regions: [String.t()] | nil
1314
}
14-
defstruct [:bucket, :key, :region]
15+
defstruct [:bucket, :key, :region, :failover_regions]
1516

1617
@opts_schema [
1718
bucket: [
@@ -28,6 +29,12 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
2829
type: :string,
2930
required: true,
3031
doc: "The AWS region of the s3 bucket."
32+
],
33+
failover_regions: [
34+
type: {:list, :string},
35+
required: false,
36+
doc:
37+
"A list of AWS regions to use if calls to the default region fail. They will be tried in order."
3138
]
3239
]
3340

@@ -50,7 +57,8 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
5057
%__MODULE__{
5158
bucket: valid_opts[:bucket],
5259
key: valid_opts[:key],
53-
region: valid_opts[:region]
60+
region: valid_opts[:region],
61+
failover_regions: valid_opts[:failover_regions]
5462
}}
5563
end
5664
end
@@ -60,7 +68,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
6068
with {:ok, versions} <- list_object_versions(state),
6169
{:ok, %{etag: etag, version_id: version}} <- find_latest(versions) do
6270
Logger.info(
63-
"found latest version of s3://#{state.bucket}/#{state.key}: #{etag} with version: #{version}"
71+
bucket: state.bucket,
72+
key: state.key,
73+
version: version,
74+
message: "Found latest version of object"
6475
)
6576

6677
{:ok, etag}
@@ -72,17 +83,32 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
7283
{:error, "could not find s3://#{state.bucket}/#{state.key}"}
7384

7485
{:error, reason} ->
75-
Logger.error("#{__MODULE__} - unknown error: #{inspect(reason)}")
86+
Logger.error(%{
87+
bucket: state.bucket,
88+
key: state.key,
89+
reason: inspect(reason),
90+
message: "Failed to get current version of object - unknown reason"
91+
})
92+
7693
{:error, "Unknown error"}
7794
end
7895
end
7996

8097
@impl true
8198
def download(state) do
82-
Logger.info("downloading s3://#{state.bucket}/#{state.key}...")
99+
Logger.info(
100+
bucket: state.bucket,
101+
key: state.key,
102+
message: "Downloading object from S3"
103+
)
83104

84105
with {:ok, %{body: body}} <- get_object(state) do
85-
Logger.debug("downloaded s3://#{state.bucket}/#{state.key}!")
106+
Logger.debug(
107+
bucket: state.bucket,
108+
key: state.key,
109+
message: "Downloaded object from S3"
110+
)
111+
86112
{:ok, body}
87113
else
88114
{:error, reason} ->
@@ -94,7 +120,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
94120
res =
95121
state.bucket
96122
|> ExAws.S3.get_bucket_object_versions(prefix: state.key)
97-
|> aws_client_request(state.region)
123+
|> aws_client_request(state)
98124

99125
with {:ok, %{body: %{versions: versions}}} <- res do
100126
{:ok, versions}
@@ -104,7 +130,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
104130
defp get_object(state) do
105131
state.bucket
106132
|> ExAws.S3.get_object(state.key)
107-
|> aws_client_request(state.region)
133+
|> aws_client_request(state)
108134
end
109135

110136
defp find_latest([_ | _] = contents) do
@@ -123,8 +149,57 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
123149

124150
defp find_latest(_), do: {:error, :not_found}
125151

126-
defp aws_client_request(op, region) do
127-
client().request(op, region: region)
152+
defp aws_client_request(op, %{region: region, failover_regions: nil}),
153+
do: client().request(op, region: region)
154+
155+
defp aws_client_request(
156+
op,
157+
%{
158+
region: region,
159+
bucket: bucket,
160+
key: key,
161+
failover_regions: failover_regions
162+
} = state
163+
)
164+
when is_list(failover_regions) do
165+
with {:error, reason} <- client().request(op, region: region) do
166+
Logger.error(%{
167+
bucket: bucket,
168+
key: key,
169+
region: region,
170+
reason: inspect(reason),
171+
message: "Failed to fetch from primary region, attempting failover regions"
172+
})
173+
174+
try_failover_regions(op, failover_regions, state)
175+
end
176+
end
177+
178+
defp try_failover_regions(_op, [], _state), do: {:error, "All regions failed"}
179+
180+
defp try_failover_regions(op, [region | remaining_regions], state) do
181+
Logger.info(%{
182+
bucket: state.bucket,
183+
key: state.key,
184+
region: region,
185+
message: "Trying failover region"
186+
})
187+
188+
case client().request(op, region: region) do
189+
{:ok, result} ->
190+
{:ok, result}
191+
192+
{:error, reason} ->
193+
Logger.error(%{
194+
bucket: state.bucket,
195+
key: state.key,
196+
region: region,
197+
reason: inspect(reason),
198+
message: "Failed to fetch from failover region"
199+
})
200+
201+
try_failover_regions(op, remaining_regions, state)
202+
end
128203
end
129204

130205
defp client, do: Application.get_env(:remote_persistent_term, :aws_client, ExAws)

‎mix.exs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ defmodule RemotePersistentTerm.MixProject do
22
use Mix.Project
33

44
@name "RemotePersistentTerm"
5-
@version "0.10.1"
5+
@version "0.11.0"
66
@repo_url "https://github.com/AppMonet/remote_persistent_term"
77

88
def project do

‎test/remote_persistent_term/fetcher/s3_test.exs‎

Lines changed: 178 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,27 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do
55
setup :verify_on_exit!
66
import ExUnit.CaptureLog
77

8+
@bucket "test-bucket"
9+
@key "test-key"
10+
@region "test-region"
11+
@failover_regions ["failover-region-1", "failover-region-2"]
12+
@version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4"
13+
814
test "Unknown error returns an error for current_version/1" do
915
expect(AwsClientMock, :request, fn _op, _opts ->
1016
{:error, :unknown_error}
1117
end)
1218

13-
assert capture_log(fn ->
14-
assert {:error, "Unknown error"} = S3.current_version(%S3{bucket: "bucket"})
15-
end) =~
16-
"Elixir.RemotePersistentTerm.Fetcher.S3 - unknown error: :unknown_error"
19+
log =
20+
capture_log(fn ->
21+
assert {:error, "Unknown error"} =
22+
S3.current_version(%S3{bucket: "bucket", key: "key"})
23+
end)
24+
25+
assert log =~ "bucket: \"bucket\""
26+
assert log =~ "key: \"key\""
27+
assert log =~ "reason: \":unknown_error\""
28+
assert log =~ "Failed to get current version of object - unknown reason"
1729
end
1830

1931
describe "init/1" do
@@ -26,4 +38,166 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do
2638
S3.init(bucket: bucket, key: key, region: region)
2739
end
2840
end
41+
42+
describe "failover_regions" do
43+
test "current_identifiers/1 tries first failover region when primary region fails" do
44+
# Setup state with failover regions
45+
state = %S3{
46+
bucket: @bucket,
47+
key: @key,
48+
region: @region,
49+
failover_regions: @failover_regions
50+
}
51+
52+
# Mock the AWS client to fail for primary region but succeed for first failover region
53+
expect(AwsClientMock, :request, 2, fn _op, opts ->
54+
case opts do
55+
[region: @region] ->
56+
{:error, "Primary region connection error"}
57+
58+
[region: "failover-region-1"] ->
59+
{:ok,
60+
%{
61+
body: %{
62+
versions: [
63+
%{version_id: @version, etag: "current-etag", is_latest: "true"}
64+
]
65+
}
66+
}}
67+
end
68+
end)
69+
70+
log =
71+
capture_log(fn ->
72+
result = S3.current_version(state)
73+
assert {:ok, "current-etag"} = result
74+
end)
75+
76+
assert log =~ "bucket: \"#{@bucket}\""
77+
assert log =~ "key: \"#{@key}\""
78+
assert log =~ "region: \"#{@region}\""
79+
assert log =~ "Failed to fetch from primary region, attempting failover regions"
80+
assert log =~ "region: \"failover-region-1\""
81+
assert log =~ "Trying failover region"
82+
assert log =~ "Found latest version of object"
83+
end
84+
85+
test "download/1 tries first failover region when primary region fails" do
86+
state = %S3{
87+
bucket: @bucket,
88+
key: @key,
89+
region: @region,
90+
failover_regions: @failover_regions
91+
}
92+
93+
# Mock the AWS client to fail for primary region but succeed for first failover region
94+
expect(AwsClientMock, :request, 2, fn _op, opts ->
95+
case opts do
96+
[region: @region] ->
97+
{:error, "Primary region connection error"}
98+
99+
[region: "failover-region-1"] ->
100+
{:ok, %{body: "content from failover region"}}
101+
end
102+
end)
103+
104+
log =
105+
capture_log(fn ->
106+
result = S3.download(state)
107+
assert {:ok, "content from failover region"} = result
108+
end)
109+
110+
assert log =~ "bucket: \"#{@bucket}\""
111+
assert log =~ "key: \"#{@key}\""
112+
assert log =~ "Downloading object from S3"
113+
assert log =~ "region: \"#{@region}\""
114+
assert log =~ "Failed to fetch from primary region, attempting failover regions"
115+
assert log =~ "region: \"failover-region-1\""
116+
assert log =~ "Trying failover region"
117+
assert log =~ "Downloaded object from S3"
118+
end
119+
120+
test "returns error when primary and all failover regions fail" do
121+
state = %S3{
122+
bucket: @bucket,
123+
key: @key,
124+
region: @region,
125+
failover_regions: @failover_regions
126+
}
127+
128+
# Mock the AWS client to fail for all regions
129+
expect(AwsClientMock, :request, 3, fn _op, opts ->
130+
case opts do
131+
[region: @region] ->
132+
{:error, "Primary region connection error"}
133+
134+
[region: "failover-region-1"] ->
135+
{:error, "First failover region connection error"}
136+
137+
[region: "failover-region-2"] ->
138+
{:error, "Second failover region connection error"}
139+
end
140+
end)
141+
142+
log =
143+
capture_log(fn ->
144+
result = S3.download(state)
145+
assert {:error, message} = result
146+
assert message =~ "All regions failed"
147+
end)
148+
149+
assert log =~ "bucket: \"#{@bucket}\""
150+
assert log =~ "key: \"#{@key}\""
151+
assert log =~ "Downloading object from S3"
152+
assert log =~ "region: \"#{@region}\""
153+
assert log =~ "Failed to fetch from primary region, attempting failover regions"
154+
assert log =~ "region: \"failover-region-1\""
155+
assert log =~ "Trying failover region"
156+
assert log =~ "reason: \"\\\"First failover region connection error\\\"\""
157+
assert log =~ "Failed to fetch from failover region"
158+
assert log =~ "region: \"failover-region-2\""
159+
assert log =~ "reason: \"\\\"Second failover region connection error\\\"\""
160+
end
161+
162+
test "tries second failover region when first failover region fails" do
163+
state = %S3{
164+
bucket: @bucket,
165+
key: @key,
166+
region: @region,
167+
failover_regions: @failover_regions
168+
}
169+
170+
# Mock the AWS client to fail for primary and first failover region but succeed for second failover region
171+
expect(AwsClientMock, :request, 3, fn _op, opts ->
172+
case opts do
173+
[region: @region] ->
174+
{:error, "Primary region connection error"}
175+
176+
[region: "failover-region-1"] ->
177+
{:error, "First failover region connection error"}
178+
179+
[region: "failover-region-2"] ->
180+
{:ok, %{body: "content from second failover region"}}
181+
end
182+
end)
183+
184+
log =
185+
capture_log(fn ->
186+
result = S3.download(state)
187+
assert {:ok, "content from second failover region"} = result
188+
end)
189+
190+
assert log =~ "bucket: \"#{@bucket}\""
191+
assert log =~ "key: \"#{@key}\""
192+
assert log =~ "Downloading object from S3"
193+
assert log =~ "region: \"#{@region}\""
194+
assert log =~ "Failed to fetch from primary region, attempting failover regions"
195+
assert log =~ "region: \"failover-region-1\""
196+
assert log =~ "Trying failover region"
197+
assert log =~ "reason: \"\\\"First failover region connection error\\\"\""
198+
assert log =~ "Failed to fetch from failover region"
199+
assert log =~ "region: \"failover-region-2\""
200+
assert log =~ "Downloaded object from S3"
201+
end
202+
end
29203
end

0 commit comments

Comments
 (0)