Skip to content

Commit 35dfa8e

Browse files
kaylareopellekaylareopelle
committed
fix: Limit length of response body read to 4mb
Limiting the read size may help prevent memory exhaustion exploits when the configured collector endpoint is attacker-controlled.
1 parent 51cf44d commit 35dfa8e

9 files changed

Lines changed: 501 additions & 27 deletions

File tree

‎exporter/otlp-http/lib/opentelemetry/exporter/otlp/http/trace_exporter.rb‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ class TraceExporter # rubocop:disable Metrics/ClassLength
2525
# Default timeouts in seconds.
2626
KEEP_ALIVE_TIMEOUT = 30
2727
RETRY_COUNT = 5
28-
private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT)
28+
RESPONSE_BODY_LIMIT = 4_194_304 # 4 MB
29+
private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :RESPONSE_BODY_LIMIT)
2930

3031
ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash'
3132
private_constant(:ERROR_MESSAGE_INVALID_HEADERS)
@@ -158,18 +159,19 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/MethodLength
158159

159160
case response
160161
when Net::HTTPSuccess
161-
response.body # Read and discard body
162+
response.read_body(nil) # Discard without reading into memory
162163
SUCCESS
163164
when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests
164-
response.body # Read and discard body
165+
response.read_body(nil) # Discard without reading into memory
165166
redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code)
166167
FAILURE
167168
when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway
168-
response.body # Read and discard body
169+
response.read_body(nil) # Discard without reading into memory
169170
redo if backoff?(retry_count: retry_count += 1, reason: response.code)
170171
FAILURE
171172
when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError
172-
log_status(response.body)
173+
body = read_response_body(response)
174+
log_status(body)
173175
@metrics_reporter.add_to_counter('otel.otlp_exporter.failure', labels: { 'reason' => response.code })
174176
FAILURE
175177
when Net::HTTPRedirection
@@ -216,14 +218,42 @@ def handle_redirect(location)
216218
end
217219

218220
def log_status(body)
221+
truncation_note = @body_truncated ? ' (body truncated due to size limit)' : ''
219222
status = Google::Rpc::Status.decode(body)
220223
details = status.details.map do |detail|
221224
klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass
222225
detail.unpack(klass_or_nil) if klass_or_nil
223226
end.compact
224-
OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}}")
227+
OpenTelemetry.handle_error(message: "OTLP exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note}")
225228
rescue StandardError => e
226-
OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::Exporter#log_status')
229+
OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::Exporter#log_status#{truncation_note}")
230+
ensure
231+
@body_truncated = false
232+
end
233+
234+
def read_response_body(response)
235+
return '' if response.nil?
236+
237+
body = String.new
238+
truncated = false
239+
240+
response.read_body do |chunk|
241+
if body.bytesize + chunk.bytesize <= RESPONSE_BODY_LIMIT
242+
body << chunk
243+
else
244+
remaining = RESPONSE_BODY_LIMIT - body.bytesize
245+
body << chunk.byteslice(0, remaining) if remaining > 0
246+
truncated = true
247+
break
248+
end
249+
end
250+
251+
body.force_encoding('UTF-8')
252+
@body_truncated = truncated
253+
body
254+
rescue StandardError => e
255+
OpenTelemetry.handle_error(exception: e, message: 'error reading response body')
256+
''
227257
end
228258

229259
def measure_request_duration

‎exporter/otlp-http/test/opentelemetry/exporter/otlp/http/trace_exporter_test.rb‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -742,4 +742,81 @@
742742
end
743743
end
744744
end
745+
746+
describe 'response body reading' do
747+
let(:exporter) { OpenTelemetry::Exporter::OTLP::HTTP::TraceExporter.new }
748+
let(:span_data) { OpenTelemetry::TestHelpers.create_span_data }
749+
750+
it 'discards body for successful responses without reading into memory' do
751+
stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 200, body: 'success body')
752+
753+
result = exporter.export([span_data])
754+
755+
_(result).must_equal(success)
756+
end
757+
758+
it 'discards body for retryable responses without reading into memory' do
759+
stub_request(:post, 'http://localhost:4318/v1/traces')
760+
.to_return(status: 503, body: 'service unavailable', headers: { 'Retry-After' => '0' })
761+
.then.to_return(status: 200)
762+
763+
result = exporter.export([span_data])
764+
765+
_(result).must_equal(success)
766+
end
767+
768+
it 'reads and parses error response body smaller than limit' do
769+
log_stream = StringIO.new
770+
logger = OpenTelemetry.logger
771+
OpenTelemetry.logger = ::Logger.new(log_stream)
772+
773+
details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: 'error details'))]
774+
status = ::Google::Rpc::Status.encode(::Google::Rpc::Status.new(code: 3, message: 'invalid argument', details: details))
775+
stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: status)
776+
777+
result = exporter.export([span_data])
778+
779+
_(result).must_equal(export_failure)
780+
_(log_stream.string).must_match(/invalid argument/)
781+
_(log_stream.string).wont_match(/truncated/)
782+
ensure
783+
OpenTelemetry.logger = logger
784+
end
785+
786+
it 'truncates error response body larger than 4 MB limit' do
787+
log_stream = StringIO.new
788+
logger = OpenTelemetry.logger
789+
OpenTelemetry.logger = ::Logger.new(log_stream)
790+
791+
# Create a body larger than 4 MB
792+
large_message = 'x' * 5_000_000 # 5 MB
793+
details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: large_message))]
794+
large_status = ::Google::Rpc::Status.new(code: 3, message: 'large error', details: details)
795+
large_body = ::Google::Rpc::Status.encode(large_status)
796+
797+
stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: large_body)
798+
799+
result = exporter.export([span_data])
800+
801+
_(result).must_equal(export_failure)
802+
_(log_stream.string).must_match(/body truncated due to size limit/)
803+
ensure
804+
OpenTelemetry.logger = logger
805+
end
806+
807+
it 'handles malformed error response body gracefully' do
808+
log_stream = StringIO.new
809+
logger = OpenTelemetry.logger
810+
OpenTelemetry.logger = ::Logger.new(log_stream)
811+
812+
stub_request(:post, 'http://localhost:4318/v1/traces').to_return(status: 400, body: 'not valid protobuf')
813+
814+
result = exporter.export([span_data])
815+
816+
_(result).must_equal(export_failure)
817+
_(log_stream.string).must_match(/unexpected error decoding rpc.Status/)
818+
ensure
819+
OpenTelemetry.logger = logger
820+
end
821+
end
745822
end

‎exporter/otlp-logs/lib/opentelemetry/exporter/otlp/logs/logs_exporter.rb‎

Lines changed: 37 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,8 @@ class LogsExporter # rubocop:disable Metrics/ClassLength
3131
# Default timeouts in seconds.
3232
KEEP_ALIVE_TIMEOUT = 30
3333
RETRY_COUNT = 5
34-
private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT)
34+
RESPONSE_BODY_LIMIT = 4_194_304 # 4 MB
35+
private_constant(:KEEP_ALIVE_TIMEOUT, :RETRY_COUNT, :RESPONSE_BODY_LIMIT)
3536

3637
ERROR_MESSAGE_INVALID_HEADERS = 'headers must be a String with comma-separated URL Encoded UTF-8 k=v pairs or a Hash'
3738
private_constant(:ERROR_MESSAGE_INVALID_HEADERS)
@@ -167,23 +168,24 @@ def send_bytes(bytes, timeout:) # rubocop:disable Metrics/CyclomaticComplexity,
167168

168169
case response
169170
when Net::HTTPSuccess
170-
response.body # Read and discard body
171+
response.read_body(nil) # Discard without reading into memory
171172
SUCCESS
172173
when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests
173-
response.body # Read and discard body
174+
response.read_body(nil) # Discard without reading into memory
174175
handle_http_error(response)
175176
redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1)
176177
FAILURE
177178
when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway
178-
response.body # Read and discard body
179+
response.read_body(nil) # Discard without reading into memory
179180
handle_http_error(response)
180181
redo if backoff?(retry_count: retry_count += 1)
181182
FAILURE
182183
when Net::HTTPNotFound
183184
handle_http_error(response)
184185
FAILURE
185186
when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError
186-
log_status(response.body)
187+
body = read_response_body(response)
188+
log_status(body)
187189
FAILURE
188190
when Net::HTTPRedirection
189191
@http.finish
@@ -234,14 +236,42 @@ def handle_redirect(location)
234236
end
235237

236238
def log_status(body)
239+
truncation_note = @body_truncated ? ' (body truncated due to size limit)' : ''
237240
status = Google::Rpc::Status.decode(body)
238241
details = status.details.map do |detail|
239242
klass_or_nil = ::Google::Protobuf::DescriptorPool.generated_pool.lookup(detail.type_name).msgclass
240243
detail.unpack(klass_or_nil) if klass_or_nil
241244
end.compact
242-
OpenTelemetry.handle_error(message: "OTLP logs exporter received rpc.Status{message=#{status.message}, details=#{details}}")
245+
OpenTelemetry.handle_error(message: "OTLP logs exporter received rpc.Status{message=#{status.message}, details=#{details}}#{truncation_note}")
243246
rescue StandardError => e
244-
OpenTelemetry.handle_error(exception: e, message: 'unexpected error decoding rpc.Status in OTLP::Exporter#log_status')
247+
OpenTelemetry.handle_error(exception: e, message: "unexpected error decoding rpc.Status in OTLP::Exporter#log_status#{truncation_note}")
248+
ensure
249+
@body_truncated = false
250+
end
251+
252+
def read_response_body(response)
253+
return '' if response.nil?
254+
255+
body = String.new
256+
truncated = false
257+
258+
response.read_body do |chunk|
259+
if body.bytesize + chunk.bytesize <= RESPONSE_BODY_LIMIT
260+
body << chunk
261+
else
262+
remaining = RESPONSE_BODY_LIMIT - body.bytesize
263+
body << chunk.byteslice(0, remaining) if remaining > 0
264+
truncated = true
265+
break
266+
end
267+
end
268+
269+
body.force_encoding('UTF-8')
270+
@body_truncated = truncated
271+
body
272+
rescue StandardError => e
273+
OpenTelemetry.handle_error(exception: e, message: 'error reading response body')
274+
''
245275
end
246276

247277
def backoff?(retry_count:, retry_after: nil) # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

‎exporter/otlp-logs/test/opentelemetry/exporter/otlp/logs_exporter_test.rb‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -955,4 +955,81 @@
955955
end
956956
end
957957
end
958+
959+
describe 'response body reading' do
960+
let(:exporter) { OpenTelemetry::Exporter::OTLP::Logs::LogsExporter.new }
961+
let(:log_record_data) { OpenTelemetry::TestHelpers.create_log_record_data }
962+
963+
it 'discards body for successful responses without reading into memory' do
964+
stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 200, body: 'success body')
965+
966+
result = exporter.export([log_record_data])
967+
968+
_(result).must_equal(SUCCESS)
969+
end
970+
971+
it 'discards body for retryable responses without reading into memory' do
972+
stub_request(:post, 'http://localhost:4318/v1/logs')
973+
.to_return(status: 503, body: 'service unavailable', headers: { 'Retry-After' => '0' })
974+
.then.to_return(status: 200)
975+
976+
result = exporter.export([log_record_data])
977+
978+
_(result).must_equal(SUCCESS)
979+
end
980+
981+
it 'reads and parses error response body smaller than limit' do
982+
log_stream = StringIO.new
983+
logger = OpenTelemetry.logger
984+
OpenTelemetry.logger = ::Logger.new(log_stream)
985+
986+
details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: 'error details'))]
987+
status = ::Google::Rpc::Status.encode(::Google::Rpc::Status.new(code: 3, message: 'invalid argument', details: details))
988+
stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 400, body: status)
989+
990+
result = exporter.export([log_record_data])
991+
992+
_(result).must_equal(FAILURE)
993+
_(log_stream.string).must_match(/invalid argument/)
994+
_(log_stream.string).wont_match(/truncated/)
995+
ensure
996+
OpenTelemetry.logger = logger
997+
end
998+
999+
it 'truncates error response body larger than 4 MB limit' do
1000+
log_stream = StringIO.new
1001+
logger = OpenTelemetry.logger
1002+
OpenTelemetry.logger = ::Logger.new(log_stream)
1003+
1004+
# Create a body larger than 4 MB
1005+
large_message = 'x' * 5_000_000 # 5 MB
1006+
details = [::Google::Protobuf::Any.pack(::Google::Protobuf::StringValue.new(value: large_message))]
1007+
large_status = ::Google::Rpc::Status.new(code: 3, message: 'large error', details: details)
1008+
large_body = ::Google::Rpc::Status.encode(large_status)
1009+
1010+
stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 400, body: large_body)
1011+
1012+
result = exporter.export([log_record_data])
1013+
1014+
_(result).must_equal(FAILURE)
1015+
_(log_stream.string).must_match(/body truncated due to size limit/)
1016+
ensure
1017+
OpenTelemetry.logger = logger
1018+
end
1019+
1020+
it 'handles malformed error response body gracefully' do
1021+
log_stream = StringIO.new
1022+
logger = OpenTelemetry.logger
1023+
OpenTelemetry.logger = ::Logger.new(log_stream)
1024+
1025+
stub_request(:post, 'http://localhost:4318/v1/logs').to_return(status: 400, body: 'not valid protobuf')
1026+
1027+
result = exporter.export([log_record_data])
1028+
1029+
_(result).must_equal(FAILURE)
1030+
_(log_stream.string).must_match(/unexpected error decoding rpc.Status/)
1031+
ensure
1032+
OpenTelemetry.logger = logger
1033+
end
1034+
end
9581035
end

‎exporter/otlp-metrics/lib/opentelemetry/exporter/otlp/metrics/metrics_exporter.rb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -120,23 +120,24 @@ def send_bytes(bytes, timeout:)
120120
response = @http.request(request)
121121
case response
122122
when Net::HTTPSuccess
123-
response.body # Read and discard body
123+
response.read_body(nil) # Discard without reading into memory
124124
SUCCESS
125125
when Net::HTTPServiceUnavailable, Net::HTTPTooManyRequests
126-
response.body # Read and discard body
126+
response.read_body(nil) # Discard without reading into memory
127127
redo if backoff?(retry_after: response['Retry-After'], retry_count: retry_count += 1, reason: response.code)
128128
OpenTelemetry.logger.warn('Net::HTTPServiceUnavailable/Net::HTTPTooManyRequests in MetricsExporter#send_bytes')
129129
FAILURE
130130
when Net::HTTPRequestTimeOut, Net::HTTPGatewayTimeOut, Net::HTTPBadGateway
131-
response.body # Read and discard body
131+
response.read_body(nil) # Discard without reading into memory
132132
redo if backoff?(retry_count: retry_count += 1, reason: response.code)
133133
OpenTelemetry.logger.warn('Net::HTTPRequestTimeOut/Net::HTTPGatewayTimeOut/Net::HTTPBadGateway in MetricsExporter#send_bytes')
134134
FAILURE
135135
when Net::HTTPNotFound
136136
OpenTelemetry.handle_error(message: "OTLP metrics_exporter received http.code=404 for uri: '#{@path}'")
137137
FAILURE
138138
when Net::HTTPBadRequest, Net::HTTPClientError, Net::HTTPServerError
139-
log_status(response.body)
139+
body = read_response_body(response)
140+
log_status(body)
140141
OpenTelemetry.logger.warn('Net::HTTPBadRequest/Net::HTTPClientError/Net::HTTPServerError in MetricsExporter#send_bytes')
141142
FAILURE
142143
when Net::HTTPRedirection

0 commit comments

Comments
 (0)