From 75f98cf013aa927740ced3329ea9eddea4295513 Mon Sep 17 00:00:00 2001 From: albertlockett Date: Fri, 3 Oct 2025 11:58:24 -0400 Subject: [PATCH 1/4] update multipart size bench and otel-arrow --- benchmarks/encodings/otelarrow/otelarrow.go | 32 ++-- benchmarks/go.mod | 51 ++--- benchmarks/go.sum | 132 ++++++++----- benchmarks/size_test.go | 101 ++++++++++ benchmarks/testutils/splitmetrics.go | 201 ++++++++++++++++++++ 5 files changed, 429 insertions(+), 88 deletions(-) create mode 100644 benchmarks/testutils/splitmetrics.go diff --git a/benchmarks/encodings/otelarrow/otelarrow.go b/benchmarks/encodings/otelarrow/otelarrow.go index 46556972..0e24cae3 100644 --- a/benchmarks/encodings/otelarrow/otelarrow.go +++ b/benchmarks/encodings/otelarrow/otelarrow.go @@ -1,15 +1,16 @@ package otelarrow import ( - v1 "github.com/open-telemetry/otel-arrow/api/experimental/arrow/v1" - "github.com/open-telemetry/otel-arrow/pkg/config" - "github.com/open-telemetry/otel-arrow/pkg/otel/arrow_record" - "github.com/open-telemetry/otel-arrow/pkg/otel/metrics/otlp" - "github.com/open-telemetry/otel-arrow/pkg/werror" + v1 "github.com/open-telemetry/otel-arrow/go/api/experimental/arrow/v1" + "github.com/open-telemetry/otel-arrow/go/pkg/config" + "github.com/open-telemetry/otel-arrow/go/pkg/otel/arrow_record" + "github.com/open-telemetry/otel-arrow/go/pkg/otel/metrics/otlp" + "github.com/open-telemetry/otel-arrow/go/pkg/werror" "go.opentelemetry.io/collector/pdata/pmetric" "google.golang.org/protobuf/proto" "github.com/splunk/stef/benchmarks/encodings" + "github.com/splunk/stef/benchmarks/testutils" ) type OtelArrowEncoding struct { @@ -106,18 +107,19 @@ func (*OtelArrowEncoding) Name() string { func (e *OtelArrowEncoding) StartMultipart(compression string) (encodings.MetricMultipartStream, error) { opts := []config.Option{} - if compression == "zstd" { - opts = append(opts, config.WithZstd()) - } else { - opts = append(opts, config.WithNoZstd()) - } + opts = append(opts, config.WithNoZstd()) arrowProducer := arrow_record.NewProducerWithOptions(opts...) - return &multipart{producer: arrowProducer}, nil + + return &multipart{ + producer: arrowProducer, + compression: compression, + }, nil } type multipart struct { - producer *arrow_record.Producer - bytes []byte + producer *arrow_record.Producer + compression string + bytes []byte } func (m *multipart) AppendPart(part pmetric.Metrics) error { @@ -129,7 +131,11 @@ func (m *multipart) AppendPart(part pmetric.Metrics) error { if err != nil { return err } + if m.compression == "zstd" { + bytes = testutils.CompressZstd(bytes) + } m.bytes = append(m.bytes, bytes...) + return nil } diff --git a/benchmarks/go.mod b/benchmarks/go.mod index 3a997238..50726f3d 100644 --- a/benchmarks/go.mod +++ b/benchmarks/go.mod @@ -5,13 +5,13 @@ go 1.24.0 require ( github.com/go-echarts/go-echarts/v2 v2.6.3 github.com/klauspost/compress v1.18.0 - github.com/open-telemetry/otel-arrow v0.31.0 + github.com/open-telemetry/otel-arrow/go v0.43.0 github.com/parquet-go/parquet-go v0.25.1 github.com/splunk/stef/go/otel v0.0.8 github.com/splunk/stef/go/pdata v0.0.0 github.com/splunk/stef/go/pkg v0.0.8 github.com/stretchr/testify v1.11.1 - go.opentelemetry.io/collector/pdata v1.19.0 + go.opentelemetry.io/collector/pdata v1.42.0 golang.org/x/text v0.29.0 google.golang.org/protobuf v1.36.9 modernc.org/b/v2 v2.1.9 @@ -19,41 +19,44 @@ require ( require ( github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect - github.com/apache/arrow/go/v17 v17.0.0 // indirect - github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/apache/arrow-go/v18 v18.2.0 // indirect + github.com/axiomhq/hyperloglog v0.2.5 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect - github.com/fxamacker/cbor/v2 v2.4.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/google/flatbuffers v24.3.25+incompatible // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/kamstrup/intmap v0.5.1 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pierrec/lz4/v4 v4.1.21 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect - go.opentelemetry.io/collector/config/configtelemetry v0.114.0 // indirect - go.opentelemetry.io/otel v1.31.0 // indirect - go.opentelemetry.io/otel/metric v1.31.0 // indirect - go.opentelemetry.io/otel/trace v1.31.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/featuregate v1.42.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect + golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 // indirect golang.org/x/mod v0.27.0 // indirect golang.org/x/net v0.43.0 // indirect golang.org/x/sync v0.17.0 // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/tools v0.36.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect - google.golang.org/grpc v1.68.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect + google.golang.org/grpc v1.75.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) @@ -64,3 +67,5 @@ replace github.com/splunk/stef/go/grpc => ../go/grpc replace github.com/splunk/stef/go/pdata => ../go/pdata replace github.com/splunk/stef/go/otel => ../go/otel + +replace github.com/open-telemetry/otel-arrow/go => ../../otel-arrow/go diff --git a/benchmarks/go.sum b/benchmarks/go.sum index 27132510..072a4960 100644 --- a/benchmarks/go.sum +++ b/benchmarks/go.sum @@ -3,81 +3,96 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/HdrHistogram/hdrhistogram-go v1.1.2 h1:5IcZpTvzydCQeHzK4Ef/D5rrSqwxob0t8PQPMybUNFM= github.com/HdrHistogram/hdrhistogram-go v1.1.2/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/apache/arrow/go/v17 v17.0.0 h1:RRR2bdqKcdbss9Gxy2NS/hK8i4LDMh23L6BbkN5+F54= -github.com/apache/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc= -github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc h1:Keo7wQ7UODUaHcEi7ltENhbAK2VgZjfat6mLy03tQzo= -github.com/axiomhq/hyperloglog v0.0.0-20230201085229-3ddf4bad03dc/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c= -github.com/brianvoe/gofakeit/v6 v6.17.0 h1:obbQTJeHfktJtiZzq0Q1bEpsNUs+yHrYlPVWt7BtmJ4= -github.com/brianvoe/gofakeit/v6 v6.17.0/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/apache/arrow-go/v18 v18.2.0 h1:QhWqpgZMKfWOniGPhbUxrHohWnooGURqL2R2Gg4SO1Q= +github.com/apache/arrow-go/v18 v18.2.0/go.mod h1:Ic/01WSwGJWRrdAZcxjBZ5hbApNJ28K96jGYaxzzGUc= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/axiomhq/hyperloglog v0.2.5 h1:Hefy3i8nAs8zAI/tDp+wE7N+Ltr8JnwiW3875pvl0N8= +github.com/axiomhq/hyperloglog v0.2.5/go.mod h1:DLUK9yIzpU5B6YFLjxTIcbHu1g4Y1WQb1m5RH3radaM= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= -github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88= -github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/go-echarts/go-echarts/v2 v2.6.3 h1:3bfcPIGzzyGe1s4fJECN0KO3F8VNYLFesRgV6iZWcHY= github.com/go-echarts/go-echarts/v2 v2.6.3/go.mod h1:Z+spPygZRIEyqod69r0WMnkN5RV3MwhYDtw601w3G8w= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= -github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kamstrup/intmap v0.5.1 h1:ENGAowczZA+PJPYYlreoqJvWgQVtAmX1l899WfYFVK0= +github.com/kamstrup/intmap v0.5.1/go.mod h1:gWUVWHKzWj8xpJVFf5GC0O26bWmv3GqdnIX/LMT6Aq4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/open-telemetry/otel-arrow v0.31.0 h1:KEWHM5XrUbuGktf17gp3Tgls0OHPyT0VtT5WEohiCC4= -github.com/open-telemetry/otel-arrow v0.31.0/go.mod h1:rEiUiCmxRT3RrtB0ZsT5LeTWJBynPCs0iBkVlMGk+E8= github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo= github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY= -github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= -github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -85,22 +100,36 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/assert v1.3.1 h1:vukIABvugfNMZMQO1ABsyQDJDTVQbn+LWSMy1ol1h6A= +github.com/zeebo/assert v1.3.1/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= -go.opentelemetry.io/collector/config/configtelemetry v0.114.0 h1:kjLeyrumge6wsX6ZIkicdNOlBXaEyW2PI2ZdVXz/rzY= -go.opentelemetry.io/collector/config/configtelemetry v0.114.0/go.mod h1:R0MBUxjSMVMIhljuDHWIygzzJWQyZHXXWIgQNxcFwhc= -go.opentelemetry.io/collector/pdata v1.19.0 h1:jmnU5R8TOCbwRr4B8sjdRxM7L5WnEKlQWX1dtLYxIbE= -go.opentelemetry.io/collector/pdata v1.19.0/go.mod h1:Ox1YVLe87cZDB/TL30i4SUz1cA5s6AM6SpFMfY61ICs= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/collector/featuregate v1.42.0 h1:uCVwumVBVex46DsG/fvgiTGuf9f53bALra7vGyKaqFI= +go.opentelemetry.io/collector/featuregate v1.42.0/go.mod h1:d0tiRzVYrytB6LkcYgz2ESFTv7OktRPQe0QEQcPt1L4= +go.opentelemetry.io/collector/pdata v1.42.0 h1:XEzisp/SNfKDcY4aRU6qrHeLzGypRUdYHjbBqkDFOO4= +go.opentelemetry.io/collector/pdata v1.42.0/go.mod h1:nnOmgf+RI/D5xYWgFPZ5nKuhf2E0Qy9Nx/mxoTvIq3k= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/slim/otlp v1.8.0 h1:afcLwp2XOeCbGrjufT1qWyruFt+6C9g5SOuymrSPUXQ= +go.opentelemetry.io/proto/slim/otlp v1.8.0/go.mod h1:Yaa5fjYm1SMCq0hG0x/87wV1MP9H5xDuG/1+AhvBcsI= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.1.0 h1:Uc+elixz922LHx5colXGi1ORbsW8DTIGM+gg+D9V7HE= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.1.0/go.mod h1:VyU6dTWBWv6h9w/+DYgSZAPMabWbPTFTuxp25sM8+s0= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.1.0 h1:i8YpvWGm/Uq1koL//bnbJ/26eV3OrKWm09+rDYo7keU= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.1.0/go.mod h1:pQ70xHY/ZVxNUBPn+qUWPl8nwai87eWdqL3M37lNi9A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -114,8 +143,8 @@ golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= -golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792 h1:R9PFI6EUdfVKgwKjZef7QIwGcBKu86OEFpJ9nUEP2l4= +golang.org/x/exp v0.0.0-20250718183923-645b1fa84792/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -140,7 +169,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -160,18 +188,18 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= -gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.68.0 h1:aHQeeJbo8zAkAa3pRzrVjZlbz6uSfeOXlJNQM0RAbz0= -google.golang.org/grpc v1.68.0/go.mod h1:fmSPC5AsjSBCK54MyHRx48kpOti1/jRfOlwEWywNjWA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI= +google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw= google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/benchmarks/size_test.go b/benchmarks/size_test.go index 0c5845c7..338a34eb 100644 --- a/benchmarks/size_test.go +++ b/benchmarks/size_test.go @@ -12,6 +12,7 @@ import ( "github.com/go-echarts/go-echarts/v2/charts" "github.com/go-echarts/go-echarts/v2/opts" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" "github.com/splunk/stef/benchmarks/encodings" @@ -256,6 +257,106 @@ func TestMetricsMultipart(t *testing.T) { } } +func TestMetricsMultipartBatched(t *testing.T) { + datasets := []struct { + name string + }{ + { + name: "hostandcollector-otelmetrics", + }, + { + name: "astronomy-otelmetrics", + }, + } + + testMultipartEncodings := []encodings.MetricMultipartEncoding{ + &otlp.OTLPEncoding{}, + &stef.STEFEncoding{}, + &stef.STEFUEncoding{}, + &otelarrow.OtelArrowEncoding{}, + } + + minBatchSizes := []int{ + 10, + 50, + 500, + 5000, + } + + compressions := []string{ + "none", + "zstd", + } + + chart.BeginSection("Size - Many Batches of varying size, Multipart Metrics") + + for _, compression := range compressions { + for _, batchSize := range minBatchSizes { + for _, dataset := range datasets { + chart.BeginChart("Dataset: "+dataset.name, t) + + fmt.Printf("%-30s %4v %11v %9v %4v\n", dataset.name, "Comp", "Batch Size", "Bytes", "Ratio") + + firstSize := 0 + for _, encoding := range testMultipartEncodings { + stream, err := encoding.StartMultipart(compression) + require.NoError(t, err) + + // Encode each part one after another and write to the same STEF stream. + // This models more closely the operation of STEF exporter in Collector. + + parts, err := testutils.ReadMultipartOTLPFile("testdata/" + dataset.name + ".zst") + require.NoError(t, err) + + var currMetricsBatch pmetric.Metrics = pmetric.NewMetrics() + for _, part := range parts { + part.ResourceMetrics().MoveAndAppendTo(currMetricsBatch.ResourceMetrics()) + + for currMetricsBatch.DataPointCount() > batchSize { + sizedBatch := testutils.SplitMetrics(batchSize, currMetricsBatch) + err := stream.AppendPart(sizedBatch) + require.NoError(t, err) + } + } + + // add last batch + if currMetricsBatch.DataPointCount() > 0 { + err := stream.AppendPart(currMetricsBatch) + require.NoError(t, err) + } + + byts, err := stream.FinishStream() + require.NoError(t, err) + + curSize := len(byts) + var delta string + if firstSize == 0 { + delta = "x 1.00" + } else { + delta = fmt.Sprintf("x %.2f", float64(firstSize)/float64(curSize)) + } + fmt.Printf( + "%-30s %4v %11v %9v %s\n", encoding.Name(), compression, batchSize, curSize, delta, + ) + if firstSize == 0 { + firstSize = curSize + } + + chart.Record( + nil, encoding.LongName(), "Size in bytes, compression="+compression, + float64(curSize), + ) + } + + chart.EndChart( + "Bytes", + charts.WithColorsOpts(opts.Colors{"#87BB62"}), + ) + } + } + } +} + func TestSTEFVeryShortFrames(t *testing.T) { input, err := testutils.ReadOTLPFile("testdata/hipstershop-otelmetrics.zst") require.NoError(t, err) diff --git a/benchmarks/testutils/splitmetrics.go b/benchmarks/testutils/splitmetrics.go new file mode 100644 index 00000000..908abf3b --- /dev/null +++ b/benchmarks/testutils/splitmetrics.go @@ -0,0 +1,201 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 +// +// borrowed from +// https://github.com/open-telemetry/opentelemetry-collector/blob/80d704deb46021176c3fc408f63496773388f3b1/processor/batchprocessor/splitmetrics.go + +package testutils + +import ( + "go.opentelemetry.io/collector/pdata/pmetric" +) + +// SplitMetrics removes metrics from the input data and returns a new data of the specified size. +func SplitMetrics(size int, src pmetric.Metrics) pmetric.Metrics { + dataPoints := src.DataPointCount() + if dataPoints <= size { + return src + } + totalCopiedDataPoints := 0 + dest := pmetric.NewMetrics() + + src.ResourceMetrics().RemoveIf(func(srcRs pmetric.ResourceMetrics) bool { + // If we are done skip everything else. + if totalCopiedDataPoints == size { + return false + } + + // If it fully fits + srcRsDataPointCount := resourceMetricsDPC(srcRs) + if (totalCopiedDataPoints + srcRsDataPointCount) <= size { + totalCopiedDataPoints += srcRsDataPointCount + srcRs.MoveTo(dest.ResourceMetrics().AppendEmpty()) + return true + } + + destRs := dest.ResourceMetrics().AppendEmpty() + srcRs.Resource().CopyTo(destRs.Resource()) + srcRs.ScopeMetrics().RemoveIf(func(srcIlm pmetric.ScopeMetrics) bool { + // If we are done skip everything else. + if totalCopiedDataPoints == size { + return false + } + + // If possible to move all metrics do that. + srcIlmDataPointCount := scopeMetricsDPC(srcIlm) + if srcIlmDataPointCount+totalCopiedDataPoints <= size { + totalCopiedDataPoints += srcIlmDataPointCount + srcIlm.MoveTo(destRs.ScopeMetrics().AppendEmpty()) + return true + } + + destIlm := destRs.ScopeMetrics().AppendEmpty() + srcIlm.Scope().CopyTo(destIlm.Scope()) + srcIlm.Metrics().RemoveIf(func(srcMetric pmetric.Metric) bool { + // If we are done skip everything else. + if totalCopiedDataPoints == size { + return false + } + + // If possible to move all points do that. + srcMetricPointCount := metricDPC(srcMetric) + if srcMetricPointCount+totalCopiedDataPoints <= size { + totalCopiedDataPoints += srcMetricPointCount + srcMetric.MoveTo(destIlm.Metrics().AppendEmpty()) + return true + } + + // If the metric has more data points than free slots we should split it. + copiedDataPoints, remove := splitMetric(srcMetric, destIlm.Metrics().AppendEmpty(), size-totalCopiedDataPoints) + totalCopiedDataPoints += copiedDataPoints + return remove + }) + return false + }) + return srcRs.ScopeMetrics().Len() == 0 + }) + + return dest +} + +// resourceMetricsDPC calculates the total number of data points in the pmetric.ResourceMetrics. +func resourceMetricsDPC(rs pmetric.ResourceMetrics) int { + dataPointCount := 0 + ilms := rs.ScopeMetrics() + for k := 0; k < ilms.Len(); k++ { + dataPointCount += scopeMetricsDPC(ilms.At(k)) + } + return dataPointCount +} + +// scopeMetricsDPC calculates the total number of data points in the pmetric.ScopeMetrics. +func scopeMetricsDPC(ilm pmetric.ScopeMetrics) int { + dataPointCount := 0 + ms := ilm.Metrics() + for k := 0; k < ms.Len(); k++ { + dataPointCount += metricDPC(ms.At(k)) + } + return dataPointCount +} + +// metricDPC calculates the total number of data points in the pmetric.Metric. +func metricDPC(ms pmetric.Metric) int { + switch ms.Type() { + case pmetric.MetricTypeGauge: + return ms.Gauge().DataPoints().Len() + case pmetric.MetricTypeSum: + return ms.Sum().DataPoints().Len() + case pmetric.MetricTypeHistogram: + return ms.Histogram().DataPoints().Len() + case pmetric.MetricTypeExponentialHistogram: + return ms.ExponentialHistogram().DataPoints().Len() + case pmetric.MetricTypeSummary: + return ms.Summary().DataPoints().Len() + } + return 0 +} + +// splitMetric removes metric points from the input data and moves data of the specified size to destination. +// Returns size of moved data and boolean describing, whether the metric should be removed from original slice. +func splitMetric(ms, dest pmetric.Metric, size int) (int, bool) { + dest.SetName(ms.Name()) + dest.SetDescription(ms.Description()) + dest.SetUnit(ms.Unit()) + + switch ms.Type() { + case pmetric.MetricTypeGauge: + return splitNumberDataPoints(ms.Gauge().DataPoints(), dest.SetEmptyGauge().DataPoints(), size) + case pmetric.MetricTypeSum: + destSum := dest.SetEmptySum() + destSum.SetAggregationTemporality(ms.Sum().AggregationTemporality()) + destSum.SetIsMonotonic(ms.Sum().IsMonotonic()) + return splitNumberDataPoints(ms.Sum().DataPoints(), destSum.DataPoints(), size) + case pmetric.MetricTypeHistogram: + destHistogram := dest.SetEmptyHistogram() + destHistogram.SetAggregationTemporality(ms.Histogram().AggregationTemporality()) + return splitHistogramDataPoints(ms.Histogram().DataPoints(), destHistogram.DataPoints(), size) + case pmetric.MetricTypeExponentialHistogram: + destHistogram := dest.SetEmptyExponentialHistogram() + destHistogram.SetAggregationTemporality(ms.ExponentialHistogram().AggregationTemporality()) + return splitExponentialHistogramDataPoints(ms.ExponentialHistogram().DataPoints(), destHistogram.DataPoints(), size) + case pmetric.MetricTypeSummary: + return splitSummaryDataPoints(ms.Summary().DataPoints(), dest.SetEmptySummary().DataPoints(), size) + } + return size, false +} + +func splitNumberDataPoints(src, dst pmetric.NumberDataPointSlice, size int) (int, bool) { + dst.EnsureCapacity(size) + i := 0 + src.RemoveIf(func(dp pmetric.NumberDataPoint) bool { + if i < size { + dp.MoveTo(dst.AppendEmpty()) + i++ + return true + } + return false + }) + return size, false +} + +func splitHistogramDataPoints(src, dst pmetric.HistogramDataPointSlice, size int) (int, bool) { + dst.EnsureCapacity(size) + i := 0 + src.RemoveIf(func(dp pmetric.HistogramDataPoint) bool { + if i < size { + dp.MoveTo(dst.AppendEmpty()) + i++ + return true + } + return false + }) + return size, false +} + +func splitExponentialHistogramDataPoints(src, dst pmetric.ExponentialHistogramDataPointSlice, size int) (int, bool) { + dst.EnsureCapacity(size) + i := 0 + src.RemoveIf(func(dp pmetric.ExponentialHistogramDataPoint) bool { + if i < size { + dp.MoveTo(dst.AppendEmpty()) + i++ + return true + } + return false + }) + return size, false +} + +func splitSummaryDataPoints(src, dst pmetric.SummaryDataPointSlice, size int) (int, bool) { + dst.EnsureCapacity(size) + i := 0 + src.RemoveIf(func(dp pmetric.SummaryDataPoint) bool { + if i < size { + dp.MoveTo(dst.AppendEmpty()) + i++ + return true + } + return false + }) + return size, false +} From 76f77e3912f828c1c89cea9a57187be9e74deb06 Mon Sep 17 00:00:00 2001 From: albertlockett Date: Fri, 3 Oct 2025 13:27:32 -0400 Subject: [PATCH 2/4] Cleanup unneeded code --- benchmarks/size_test.go | 101 -------------- benchmarks/testutils/splitmetrics.go | 201 --------------------------- 2 files changed, 302 deletions(-) delete mode 100644 benchmarks/testutils/splitmetrics.go diff --git a/benchmarks/size_test.go b/benchmarks/size_test.go index 338a34eb..0c5845c7 100644 --- a/benchmarks/size_test.go +++ b/benchmarks/size_test.go @@ -12,7 +12,6 @@ import ( "github.com/go-echarts/go-echarts/v2/charts" "github.com/go-echarts/go-echarts/v2/opts" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" "github.com/splunk/stef/benchmarks/encodings" @@ -257,106 +256,6 @@ func TestMetricsMultipart(t *testing.T) { } } -func TestMetricsMultipartBatched(t *testing.T) { - datasets := []struct { - name string - }{ - { - name: "hostandcollector-otelmetrics", - }, - { - name: "astronomy-otelmetrics", - }, - } - - testMultipartEncodings := []encodings.MetricMultipartEncoding{ - &otlp.OTLPEncoding{}, - &stef.STEFEncoding{}, - &stef.STEFUEncoding{}, - &otelarrow.OtelArrowEncoding{}, - } - - minBatchSizes := []int{ - 10, - 50, - 500, - 5000, - } - - compressions := []string{ - "none", - "zstd", - } - - chart.BeginSection("Size - Many Batches of varying size, Multipart Metrics") - - for _, compression := range compressions { - for _, batchSize := range minBatchSizes { - for _, dataset := range datasets { - chart.BeginChart("Dataset: "+dataset.name, t) - - fmt.Printf("%-30s %4v %11v %9v %4v\n", dataset.name, "Comp", "Batch Size", "Bytes", "Ratio") - - firstSize := 0 - for _, encoding := range testMultipartEncodings { - stream, err := encoding.StartMultipart(compression) - require.NoError(t, err) - - // Encode each part one after another and write to the same STEF stream. - // This models more closely the operation of STEF exporter in Collector. - - parts, err := testutils.ReadMultipartOTLPFile("testdata/" + dataset.name + ".zst") - require.NoError(t, err) - - var currMetricsBatch pmetric.Metrics = pmetric.NewMetrics() - for _, part := range parts { - part.ResourceMetrics().MoveAndAppendTo(currMetricsBatch.ResourceMetrics()) - - for currMetricsBatch.DataPointCount() > batchSize { - sizedBatch := testutils.SplitMetrics(batchSize, currMetricsBatch) - err := stream.AppendPart(sizedBatch) - require.NoError(t, err) - } - } - - // add last batch - if currMetricsBatch.DataPointCount() > 0 { - err := stream.AppendPart(currMetricsBatch) - require.NoError(t, err) - } - - byts, err := stream.FinishStream() - require.NoError(t, err) - - curSize := len(byts) - var delta string - if firstSize == 0 { - delta = "x 1.00" - } else { - delta = fmt.Sprintf("x %.2f", float64(firstSize)/float64(curSize)) - } - fmt.Printf( - "%-30s %4v %11v %9v %s\n", encoding.Name(), compression, batchSize, curSize, delta, - ) - if firstSize == 0 { - firstSize = curSize - } - - chart.Record( - nil, encoding.LongName(), "Size in bytes, compression="+compression, - float64(curSize), - ) - } - - chart.EndChart( - "Bytes", - charts.WithColorsOpts(opts.Colors{"#87BB62"}), - ) - } - } - } -} - func TestSTEFVeryShortFrames(t *testing.T) { input, err := testutils.ReadOTLPFile("testdata/hipstershop-otelmetrics.zst") require.NoError(t, err) diff --git a/benchmarks/testutils/splitmetrics.go b/benchmarks/testutils/splitmetrics.go deleted file mode 100644 index 908abf3b..00000000 --- a/benchmarks/testutils/splitmetrics.go +++ /dev/null @@ -1,201 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 -// -// borrowed from -// https://github.com/open-telemetry/opentelemetry-collector/blob/80d704deb46021176c3fc408f63496773388f3b1/processor/batchprocessor/splitmetrics.go - -package testutils - -import ( - "go.opentelemetry.io/collector/pdata/pmetric" -) - -// SplitMetrics removes metrics from the input data and returns a new data of the specified size. -func SplitMetrics(size int, src pmetric.Metrics) pmetric.Metrics { - dataPoints := src.DataPointCount() - if dataPoints <= size { - return src - } - totalCopiedDataPoints := 0 - dest := pmetric.NewMetrics() - - src.ResourceMetrics().RemoveIf(func(srcRs pmetric.ResourceMetrics) bool { - // If we are done skip everything else. - if totalCopiedDataPoints == size { - return false - } - - // If it fully fits - srcRsDataPointCount := resourceMetricsDPC(srcRs) - if (totalCopiedDataPoints + srcRsDataPointCount) <= size { - totalCopiedDataPoints += srcRsDataPointCount - srcRs.MoveTo(dest.ResourceMetrics().AppendEmpty()) - return true - } - - destRs := dest.ResourceMetrics().AppendEmpty() - srcRs.Resource().CopyTo(destRs.Resource()) - srcRs.ScopeMetrics().RemoveIf(func(srcIlm pmetric.ScopeMetrics) bool { - // If we are done skip everything else. - if totalCopiedDataPoints == size { - return false - } - - // If possible to move all metrics do that. - srcIlmDataPointCount := scopeMetricsDPC(srcIlm) - if srcIlmDataPointCount+totalCopiedDataPoints <= size { - totalCopiedDataPoints += srcIlmDataPointCount - srcIlm.MoveTo(destRs.ScopeMetrics().AppendEmpty()) - return true - } - - destIlm := destRs.ScopeMetrics().AppendEmpty() - srcIlm.Scope().CopyTo(destIlm.Scope()) - srcIlm.Metrics().RemoveIf(func(srcMetric pmetric.Metric) bool { - // If we are done skip everything else. - if totalCopiedDataPoints == size { - return false - } - - // If possible to move all points do that. - srcMetricPointCount := metricDPC(srcMetric) - if srcMetricPointCount+totalCopiedDataPoints <= size { - totalCopiedDataPoints += srcMetricPointCount - srcMetric.MoveTo(destIlm.Metrics().AppendEmpty()) - return true - } - - // If the metric has more data points than free slots we should split it. - copiedDataPoints, remove := splitMetric(srcMetric, destIlm.Metrics().AppendEmpty(), size-totalCopiedDataPoints) - totalCopiedDataPoints += copiedDataPoints - return remove - }) - return false - }) - return srcRs.ScopeMetrics().Len() == 0 - }) - - return dest -} - -// resourceMetricsDPC calculates the total number of data points in the pmetric.ResourceMetrics. -func resourceMetricsDPC(rs pmetric.ResourceMetrics) int { - dataPointCount := 0 - ilms := rs.ScopeMetrics() - for k := 0; k < ilms.Len(); k++ { - dataPointCount += scopeMetricsDPC(ilms.At(k)) - } - return dataPointCount -} - -// scopeMetricsDPC calculates the total number of data points in the pmetric.ScopeMetrics. -func scopeMetricsDPC(ilm pmetric.ScopeMetrics) int { - dataPointCount := 0 - ms := ilm.Metrics() - for k := 0; k < ms.Len(); k++ { - dataPointCount += metricDPC(ms.At(k)) - } - return dataPointCount -} - -// metricDPC calculates the total number of data points in the pmetric.Metric. -func metricDPC(ms pmetric.Metric) int { - switch ms.Type() { - case pmetric.MetricTypeGauge: - return ms.Gauge().DataPoints().Len() - case pmetric.MetricTypeSum: - return ms.Sum().DataPoints().Len() - case pmetric.MetricTypeHistogram: - return ms.Histogram().DataPoints().Len() - case pmetric.MetricTypeExponentialHistogram: - return ms.ExponentialHistogram().DataPoints().Len() - case pmetric.MetricTypeSummary: - return ms.Summary().DataPoints().Len() - } - return 0 -} - -// splitMetric removes metric points from the input data and moves data of the specified size to destination. -// Returns size of moved data and boolean describing, whether the metric should be removed from original slice. -func splitMetric(ms, dest pmetric.Metric, size int) (int, bool) { - dest.SetName(ms.Name()) - dest.SetDescription(ms.Description()) - dest.SetUnit(ms.Unit()) - - switch ms.Type() { - case pmetric.MetricTypeGauge: - return splitNumberDataPoints(ms.Gauge().DataPoints(), dest.SetEmptyGauge().DataPoints(), size) - case pmetric.MetricTypeSum: - destSum := dest.SetEmptySum() - destSum.SetAggregationTemporality(ms.Sum().AggregationTemporality()) - destSum.SetIsMonotonic(ms.Sum().IsMonotonic()) - return splitNumberDataPoints(ms.Sum().DataPoints(), destSum.DataPoints(), size) - case pmetric.MetricTypeHistogram: - destHistogram := dest.SetEmptyHistogram() - destHistogram.SetAggregationTemporality(ms.Histogram().AggregationTemporality()) - return splitHistogramDataPoints(ms.Histogram().DataPoints(), destHistogram.DataPoints(), size) - case pmetric.MetricTypeExponentialHistogram: - destHistogram := dest.SetEmptyExponentialHistogram() - destHistogram.SetAggregationTemporality(ms.ExponentialHistogram().AggregationTemporality()) - return splitExponentialHistogramDataPoints(ms.ExponentialHistogram().DataPoints(), destHistogram.DataPoints(), size) - case pmetric.MetricTypeSummary: - return splitSummaryDataPoints(ms.Summary().DataPoints(), dest.SetEmptySummary().DataPoints(), size) - } - return size, false -} - -func splitNumberDataPoints(src, dst pmetric.NumberDataPointSlice, size int) (int, bool) { - dst.EnsureCapacity(size) - i := 0 - src.RemoveIf(func(dp pmetric.NumberDataPoint) bool { - if i < size { - dp.MoveTo(dst.AppendEmpty()) - i++ - return true - } - return false - }) - return size, false -} - -func splitHistogramDataPoints(src, dst pmetric.HistogramDataPointSlice, size int) (int, bool) { - dst.EnsureCapacity(size) - i := 0 - src.RemoveIf(func(dp pmetric.HistogramDataPoint) bool { - if i < size { - dp.MoveTo(dst.AppendEmpty()) - i++ - return true - } - return false - }) - return size, false -} - -func splitExponentialHistogramDataPoints(src, dst pmetric.ExponentialHistogramDataPointSlice, size int) (int, bool) { - dst.EnsureCapacity(size) - i := 0 - src.RemoveIf(func(dp pmetric.ExponentialHistogramDataPoint) bool { - if i < size { - dp.MoveTo(dst.AppendEmpty()) - i++ - return true - } - return false - }) - return size, false -} - -func splitSummaryDataPoints(src, dst pmetric.SummaryDataPointSlice, size int) (int, bool) { - dst.EnsureCapacity(size) - i := 0 - src.RemoveIf(func(dp pmetric.SummaryDataPoint) bool { - if i < size { - dp.MoveTo(dst.AppendEmpty()) - i++ - return true - } - return false - }) - return size, false -} From 772a429d345a3e3553f2ca5dc8dcf0a9c6247a33 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Fri, 3 Oct 2025 14:27:03 -0400 Subject: [PATCH 3/4] Fix go.mod issues --- benchmarks/go.mod | 2 -- benchmarks/go.sum | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/go.mod b/benchmarks/go.mod index 50726f3d..60b31108 100644 --- a/benchmarks/go.mod +++ b/benchmarks/go.mod @@ -67,5 +67,3 @@ replace github.com/splunk/stef/go/grpc => ../go/grpc replace github.com/splunk/stef/go/pdata => ../go/pdata replace github.com/splunk/stef/go/otel => ../go/otel - -replace github.com/open-telemetry/otel-arrow/go => ../../otel-arrow/go diff --git a/benchmarks/go.sum b/benchmarks/go.sum index 072a4960..49d7a7a1 100644 --- a/benchmarks/go.sum +++ b/benchmarks/go.sum @@ -82,6 +82,8 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/open-telemetry/otel-arrow/go v0.43.0 h1:P1BsergAwmn4C5WbuQaS4S3cHIaOirJlTbsUtUPiPNY= +github.com/open-telemetry/otel-arrow/go v0.43.0/go.mod h1:gGW+5QC51n86Vj3F3FY9m8hTfb6llZrDhW+8HB/mBGw= github.com/parquet-go/parquet-go v0.25.1 h1:l7jJwNM0xrk0cnIIptWMtnSnuxRkwq53S+Po3KG8Xgo= github.com/parquet-go/parquet-go v0.25.1/go.mod h1:AXBuotO1XiBtcqJb/FKFyjBG4aqa3aQAAWF3ZPzCanY= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= From dea89f884f05a02af44556adec2939e0f74128c9 Mon Sep 17 00:00:00 2001 From: Tigran Najaryan Date: Fri, 3 Oct 2025 15:56:44 -0400 Subject: [PATCH 4/4] Re-run and update benchmarks.html --- docs/benchmarks.html | 120 +++++++++++++++++++++---------------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/docs/benchmarks.html b/docs/benchmarks.html index 1a64102f..424b78d7 100644 --- a/docs/benchmarks.html +++ b/docs/benchmarks.html @@ -9,186 +9,186 @@

Size Benchmarks - One Large Batch

-
+
-
+
-
+

Size - Many Batches, Multipart Metrics

-
+
-
+
-
+
-
+

Size - Many Batches, Multipart Traces

-
+
-
+
-
+
-
+

Speed Benchmarks

-
+
-
+
-
+
-
+
\ No newline at end of file