diff --git a/cmd/otelcorecol/builder-config.yaml b/cmd/otelcorecol/builder-config.yaml index a0dd3029a1f..303b8d29caf 100644 --- a/cmd/otelcorecol/builder-config.yaml +++ b/cmd/otelcorecol/builder-config.yaml @@ -22,6 +22,8 @@ exporters: - gomod: go.opentelemetry.io/collector/exporter/otlphttpexporter v0.124.0 extensions: - gomod: go.opentelemetry.io/collector/extension/memorylimiterextension v0.124.0 + - gomod: go.opentelemetry.io/collector/extension/limitermiddlewareextension v0.124.0 + - gomod: go.opentelemetry.io/collector/extension/ratelimiterextension v0.124.0 - gomod: go.opentelemetry.io/collector/extension/zpagesextension v0.124.0 processors: - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.124.0 @@ -46,6 +48,7 @@ replaces: - go.opentelemetry.io/collector/config/configcompression => ../../config/configcompression - go.opentelemetry.io/collector/config/configgrpc => ../../config/configgrpc - go.opentelemetry.io/collector/config/confighttp => ../../config/confighttp + - go.opentelemetry.io/collector/config/configlimiter => ../../config/configlimiter - go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware - go.opentelemetry.io/collector/config/confignet => ../../config/confignet - go.opentelemetry.io/collector/config/configopaque => ../../config/configopaque @@ -80,10 +83,13 @@ replaces: - go.opentelemetry.io/collector/extension/extensionauth => ../../extension/extensionauth - go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest => ../../extension/extensionauth/extensionauthtest - go.opentelemetry.io/collector/extension/extensioncapabilities => ../../extension/extensioncapabilities + - go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter - go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware - go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest - go.opentelemetry.io/collector/extension/extensiontest => ../../extension/extensiontest - go.opentelemetry.io/collector/extension/memorylimiterextension => ../../extension/memorylimiterextension + - go.opentelemetry.io/collector/extension/ratelimiterextension => ../../extension/ratelimiterextension + - go.opentelemetry.io/collector/extension/limitermiddlewareextension => ../../extension/limitermiddlewareextension - go.opentelemetry.io/collector/extension/xextension => ../../extension/xextension - go.opentelemetry.io/collector/extension/zpagesextension => ../../extension/zpagesextension - go.opentelemetry.io/collector/featuregate => ../../featuregate diff --git a/cmd/otelcorecol/components.go b/cmd/otelcorecol/components.go index 18754860959..cbd08f7492b 100644 --- a/cmd/otelcorecol/components.go +++ b/cmd/otelcorecol/components.go @@ -12,7 +12,9 @@ import ( otlpexporter "go.opentelemetry.io/collector/exporter/otlpexporter" otlphttpexporter "go.opentelemetry.io/collector/exporter/otlphttpexporter" "go.opentelemetry.io/collector/extension" + limitermiddlewareextension "go.opentelemetry.io/collector/extension/limitermiddlewareextension" memorylimiterextension "go.opentelemetry.io/collector/extension/memorylimiterextension" + ratelimiterextension "go.opentelemetry.io/collector/extension/ratelimiterextension" zpagesextension "go.opentelemetry.io/collector/extension/zpagesextension" "go.opentelemetry.io/collector/otelcol" "go.opentelemetry.io/collector/processor" @@ -29,6 +31,8 @@ func components() (otelcol.Factories, error) { factories.Extensions, err = otelcol.MakeFactoryMap[extension.Factory]( memorylimiterextension.NewFactory(), + limitermiddlewareextension.NewFactory(), + ratelimiterextension.NewFactory(), zpagesextension.NewFactory(), ) if err != nil { @@ -36,6 +40,8 @@ func components() (otelcol.Factories, error) { } factories.ExtensionModules = make(map[component.Type]string, len(factories.Extensions)) factories.ExtensionModules[memorylimiterextension.NewFactory().Type()] = "go.opentelemetry.io/collector/extension/memorylimiterextension v0.124.0" + factories.ExtensionModules[limitermiddlewareextension.NewFactory().Type()] = "go.opentelemetry.io/collector/extension/limitermiddlewareextension v0.124.0" + factories.ExtensionModules[ratelimiterextension.NewFactory().Type()] = "go.opentelemetry.io/collector/extension/ratelimiterextension v0.124.0" factories.ExtensionModules[zpagesextension.NewFactory().Type()] = "go.opentelemetry.io/collector/extension/zpagesextension v0.124.0" factories.Receivers, err = otelcol.MakeFactoryMap[receiver.Factory]( diff --git a/cmd/otelcorecol/go.mod b/cmd/otelcorecol/go.mod index 75241b1b7e1..f1b75f68326 100644 --- a/cmd/otelcorecol/go.mod +++ b/cmd/otelcorecol/go.mod @@ -4,7 +4,7 @@ module go.opentelemetry.io/collector/cmd/otelcorecol go 1.23.0 -toolchain go1.23.8 +toolchain go1.24.1 require ( go.opentelemetry.io/collector/component v1.30.0 @@ -22,7 +22,9 @@ require ( go.opentelemetry.io/collector/exporter/otlpexporter v0.124.0 go.opentelemetry.io/collector/exporter/otlphttpexporter v0.124.0 go.opentelemetry.io/collector/extension v1.30.0 + go.opentelemetry.io/collector/extension/limitermiddlewareextension v0.124.0 go.opentelemetry.io/collector/extension/memorylimiterextension v0.124.0 + go.opentelemetry.io/collector/extension/ratelimiterextension v0.124.0 go.opentelemetry.io/collector/extension/zpagesextension v0.124.0 go.opentelemetry.io/collector/otelcol v0.124.0 go.opentelemetry.io/collector/processor v1.30.0 @@ -89,6 +91,7 @@ require ( go.opentelemetry.io/collector/config/configcompression v1.30.0 // indirect go.opentelemetry.io/collector/config/configgrpc v0.124.0 // indirect go.opentelemetry.io/collector/config/confighttp v0.124.0 // indirect + go.opentelemetry.io/collector/config/configlimiter v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/config/configmiddleware v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/config/confignet v1.30.0 // indirect go.opentelemetry.io/collector/config/configopaque v1.30.0 // indirect @@ -108,6 +111,7 @@ require ( go.opentelemetry.io/collector/exporter/xexporter v0.124.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.30.0 // indirect go.opentelemetry.io/collector/extension/extensioncapabilities v0.124.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect go.opentelemetry.io/collector/extension/extensiontest v0.124.0 // indirect go.opentelemetry.io/collector/extension/xextension v0.124.0 // indirect @@ -188,6 +192,8 @@ replace go.opentelemetry.io/collector/config/configgrpc => ../../config/configgr replace go.opentelemetry.io/collector/config/confighttp => ../../config/confighttp +replace go.opentelemetry.io/collector/config/configlimiter => ../../config/configlimiter + replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware replace go.opentelemetry.io/collector/config/confignet => ../../config/confignet @@ -256,6 +262,8 @@ replace go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest replace go.opentelemetry.io/collector/extension/extensioncapabilities => ../../extension/extensioncapabilities +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest @@ -264,6 +272,10 @@ replace go.opentelemetry.io/collector/extension/extensiontest => ../../extension replace go.opentelemetry.io/collector/extension/memorylimiterextension => ../../extension/memorylimiterextension +replace go.opentelemetry.io/collector/extension/ratelimiterextension => ../../extension/ratelimiterextension + +replace go.opentelemetry.io/collector/extension/limitermiddlewareextension => ../../extension/limitermiddlewareextension + replace go.opentelemetry.io/collector/extension/xextension => ../../extension/xextension replace go.opentelemetry.io/collector/extension/zpagesextension => ../../extension/zpagesextension diff --git a/config/configlimiter/Makefile b/config/configlimiter/Makefile new file mode 100644 index 00000000000..ded7a36092d --- /dev/null +++ b/config/configlimiter/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/config/configlimiter/configlimiter.go b/config/configlimiter/configlimiter.go new file mode 100644 index 00000000000..be0818f02ef --- /dev/null +++ b/config/configlimiter/configlimiter.go @@ -0,0 +1,42 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +// Package configlimiter implements the configuration settings to +// apply rate limiting on incoming requests, and allows +// components to configure rate limiting behavior. +package configlimiter // import "go.opentelemetry.io/collector/config/configlimiter" + +import ( + "context" + "errors" + "fmt" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +var ( + errLimiterNotFound = errors.New("limiter not found") + errNotLimiter = errors.New("requested extension is not a limiter") +) + +// Config defines the rate limiting settings for a component. +type Config struct { + // ID specifies the name of the extension to use in order to apply rate limiting. + ID component.ID `mapstructure:"id,omitempty"` +} + +// GetProvider attempts to select the appropriate extensionlimiter.Provider from the list of extensions, +// based on the requested extension name. If a limiter is not found, an error is returned. +// Callers will use the returned Provider to get access to the specific rate- and +// resource-limiter weights they are capable of limiting. +func (c Config) GetProvider(_ context.Context, extensions map[component.ID]component.Component) (extensionlimiter.Provider, error) { + if ext, found := extensions[c.ID]; found { + if limiter, ok := ext.(extensionlimiter.Provider); ok { + return limiter, nil + } + return nil, errNotLimiter + } + + return nil, fmt.Errorf("failed to resolve limiter provider %q: %w", c.ID, errLimiterNotFound) +} diff --git a/config/configlimiter/go.mod b/config/configlimiter/go.mod new file mode 100644 index 00000000000..e3d227ed0f6 --- /dev/null +++ b/config/configlimiter/go.mod @@ -0,0 +1,60 @@ +module go.opentelemetry.io/collector/config/configlimiter + +go 1.23.0 + +require ( + go.opentelemetry.io/collector/component v1.30.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 +) + +require ( + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/featuregate v1.30.0 // indirect + go.opentelemetry.io/collector/internal/telemetry v0.124.0 // indirect + go.opentelemetry.io/collector/pdata v1.30.0 // indirect + go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/log v0.11.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/grpc v1.71.1 // indirect + google.golang.org/protobuf v1.36.6 // indirect +) + +replace go.opentelemetry.io/collector/component => ../../component + +replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/pdata => ../../pdata + +replace go.opentelemetry.io/collector/pipeline => ../../pipeline + +replace go.opentelemetry.io/collector/featuregate => ../../featuregate + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/consumer => ../../consumer + +replace go.opentelemetry.io/collector/config/configmiddleware => ../configmiddleware + +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension => ../../extension diff --git a/config/configlimiter/go.sum b/config/configlimiter/go.sum new file mode 100644 index 00000000000..c0169d210e9 --- /dev/null +++ b/config/configlimiter/go.sum @@ -0,0 +1,86 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y= +go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/exporter/otlpexporter/go.mod b/exporter/otlpexporter/go.mod index 5da56342d8a..6f11853a024 100644 --- a/exporter/otlpexporter/go.mod +++ b/exporter/otlpexporter/go.mod @@ -166,8 +166,8 @@ replace go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry -replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware - replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware + replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest diff --git a/extension/extensionlimiter/Makefile b/extension/extensionlimiter/Makefile new file mode 100644 index 00000000000..ded7a36092d --- /dev/null +++ b/extension/extensionlimiter/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/extension/extensionlimiter/extensionlimiter.go b/extension/extensionlimiter/extensionlimiter.go new file mode 100644 index 00000000000..2b8fe144d0a --- /dev/null +++ b/extension/extensionlimiter/extensionlimiter.go @@ -0,0 +1,116 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package extensionlimiter // import "go.opentelemetry.io/collector/extension/extensionlimiter" + +import ( + "context" +) + +// WeightKey is an enum type for common rate limits +type WeightKey string + +// Predefined weight keys for common rate limits. This is not a closed set. +// +// Providers should return errors when they do not recognize a weight key. +const ( + // WeightKeyNetworkBytes is typically used with RateLimiters + // for limiting arrival rate. + WeightKeyNetworkBytes WeightKey = "network_bytes" + + // WeightKeyRequestCount can be used to limit the rate or + // total concurrent number of requests (i.e., pipeline data + // objects). + WeightKeyRequestCount WeightKey = "request_count" + + // WeightKeyRequestItems can be used to limit the rate or + // total concurrent number of items (log records, metric data + // points, spans, profiles). + WeightKeyRequestItems WeightKey = "request_items" + + // WeightKeyMemorySize is typically used with ResourceLimiters + // for limiting active memory usage. + WeightKeyMemorySize WeightKey = "memory_size" +) + +// Provider is an interface that provides access to different limiter types +// for specific weight keys. +// +// Extensions implementing this interface can be referenced by their +// names from component rate limiting configurations (e.g., limitermiddleware). +type Provider interface { + // RateLimiter returns a RateLimiter for the specified weight key. + // + // Rate limiters are useful for limiting an amount of + // something per unit of time. In the OpenTelemetry metrics + // data model, resource limiters would apply to value of an + // Counter, where the value is incremented on Limit(). + RateLimiter(key WeightKey) RateLimiter + + // ResourceLimiter returns a ResourceLimiter for the specified + // weight key. + // + // Resource limiters are useful for limiting a total amount of + // something. In the OpenTelemetry metrics data model, + // resource limiters would apply to value of an UpDownCounter, + // where the value is incremented on Acquire() and decremented + // in Release(). + // + // In cases where a component supports a rate limiter and does not use + // a release function, the component may return a ResourceLimiterFunc + // which calls the underlying rate limiter and returns a no-op ReleaseFunc. + ResourceLimiter(key WeightKey) ResourceLimiter +} + +// ResourceLimiter is an interface that components can use to apply +// resource limiting (e.g., concurrent requests, memory in use). +type ResourceLimiter interface { + // Acquire attempts to acquire resources based on the provided weight value. + // + // It may block until resources are available or return an error if the limit + // cannot be satisfied. + // + // On success, it returns a ReleaseFunc that should be called + // when the resources are no longer needed. + Acquire(ctx context.Context, value uint64) (ReleaseFunc, error) +} + +var _ ResourceLimiter = ResourceLimiterFunc(nil) + +// ReleaseFunc is called when resources should be released after limiting. +// +// RelaseFunc values are never nil values, even in the error case, for +// safety. Users should unconditionally defer these. +type ReleaseFunc func() + +// ResourceLimiterFunc is an easy way to construct ResourceLimiters. +type ResourceLimiterFunc func(ctx context.Context, value uint64) (ReleaseFunc, error) + +// Acquire implements ResourceLimiter. +func (f ResourceLimiterFunc) Acquire(ctx context.Context, value uint64) (ReleaseFunc, error) { + if f == nil { + return func() {}, nil + } + return f(ctx, value) +} + +// RateLimiter is an interface that components can use to apply +// rate limiting (e.g., network-bytes-per-second, requests-per-second). +type RateLimiter interface { + // Limit attempts to apply rate limiting based on the provided weight value. + // Limit is expected to block the caller until the weight can be admitted. + Limit(ctx context.Context, value uint64) error +} + +var _ RateLimiter = RateLimiterFunc(nil) + +// RateLimiterFunc is an easy way to construct RateLimiters. +type RateLimiterFunc func(ctx context.Context, value uint64) error + +// Limit implements RateLimiter. +func (f RateLimiterFunc) Limit(ctx context.Context, value uint64) error { + if f == nil { + return nil + } + return f(ctx, value) +} diff --git a/extension/extensionlimiter/go.mod b/extension/extensionlimiter/go.mod new file mode 100644 index 00000000000..67a478d159b --- /dev/null +++ b/extension/extensionlimiter/go.mod @@ -0,0 +1,65 @@ +module go.opentelemetry.io/collector/extension/extensionlimiter + +go 1.23.0 + +require ( + go.opentelemetry.io/collector/component v1.30.0 + go.opentelemetry.io/collector/config/configmiddleware v0.0.0-00010101000000-000000000000 + go.opentelemetry.io/collector/consumer v1.30.0 + go.opentelemetry.io/collector/consumer/xconsumer v0.124.0 + go.opentelemetry.io/collector/pdata v1.30.0 + go.opentelemetry.io/collector/pdata/pprofile v0.124.0 +) + +require ( + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect + go.opentelemetry.io/collector/featuregate v1.30.0 // indirect + go.opentelemetry.io/collector/internal/telemetry v0.124.0 // indirect + go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/log v0.11.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/grpc v1.71.1 // indirect + google.golang.org/protobuf v1.36.6 // indirect +) + +replace go.opentelemetry.io/collector/consumer => ../../consumer + +replace go.opentelemetry.io/collector/featuregate => ../../featuregate + +replace go.opentelemetry.io/collector/component => ../../component + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/pdata => ../../pdata + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware + +replace go.opentelemetry.io/collector/pipeline => ../../pipeline + +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/extension => ../ + +replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry diff --git a/extension/extensionlimiter/go.sum b/extension/extensionlimiter/go.sum new file mode 100644 index 00000000000..1750b23c9cc --- /dev/null +++ b/extension/extensionlimiter/go.sum @@ -0,0 +1,97 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y= +go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/extension/extensionlimiter/limiterhelper/consumer.go b/extension/extensionlimiter/limiterhelper/consumer.go new file mode 100644 index 00000000000..962edcd7a88 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/consumer.go @@ -0,0 +1,327 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + +import ( + "context" + + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/consumer/xconsumer" + "go.opentelemetry.io/collector/extension/extensionlimiter" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" + "go.opentelemetry.io/collector/pdata/pprofile" + "go.opentelemetry.io/collector/pdata/ptrace" +) + +// Consumer is a builder for creating wrapped consumers with resource limiters +// +// This supports limiting by request_count, request_items, and +// memory_size weight keys. +// +// The network_bytes weight key not supported because that information +// is not available from the pdata object. +type limiter struct { + requestItemsLimiter extensionlimiter.ResourceLimiter + memorySizeLimiter extensionlimiter.ResourceLimiter + requestCountLimiter extensionlimiter.ResourceLimiter +} + +// config stores configuration from Options. +type config struct { + requestItemsLimiter bool + memorySizeLimiter bool + requestCountLimiter bool +} + +// Option represents the consumer options +type Option interface { + apply(*config) +} + +type optionFunc func(*config) + +func (of optionFunc) apply(cfg *config) { + of(cfg) +} + +// WithRequestCountLimit configures the consumer to limit based on request count. +func WithRequestCountLimit() Option { + return optionFunc(func(c *config) { + c.requestCountLimiter = true + }) +} + +// WithRequestItemsLimit configures the consumer to limit based on item counts. +func WithRequestItemsLimit() Option { + return optionFunc(func(c *config) { + c.requestItemsLimiter = true + }) +} + +// WithMemorySizeLimit configures the consumer to limit based on memory size. +func WithMemorySizeLimit() Option { + return optionFunc(func(c *config) { + c.memorySizeLimiter = true + }) +} + +func newLimiter(provider extensionlimiter.Provider, options ...Option) *limiter { + cfg := &config{} + for _, option := range options { + option.apply(cfg) + } + c := &limiter{} + if cfg.requestCountLimiter { + c.requestCountLimiter = provider.ResourceLimiter(extensionlimiter.WeightKeyRequestCount) + } + if cfg.requestItemsLimiter { + c.requestItemsLimiter = provider.ResourceLimiter(extensionlimiter.WeightKeyRequestItems) + } + if cfg.memorySizeLimiter { + c.memorySizeLimiter = provider.ResourceLimiter(extensionlimiter.WeightKeyMemorySize) + } + return c +} + +// WrapTraces wraps a traces consumer with resource limiters +func WrapTraces(provider extensionlimiter.Provider, nextConsumer consumer.Traces, options ...Option) consumer.Traces { + limiter := newLimiter(provider, options...) + + if limiter.requestItemsLimiter == nil && limiter.memorySizeLimiter == nil && limiter.requestCountLimiter == nil { + return nextConsumer + } + return &tracesConsumer{ + nextConsumer: nextConsumer, + limiter: limiter, + } +} + +// WrapMetrics wraps a metrics consumer with resource limiters +func WrapMetrics(provider extensionlimiter.Provider, nextConsumer consumer.Metrics, options ...Option) consumer.Metrics { + limiter := newLimiter(provider, options...) + + if limiter.requestItemsLimiter == nil && limiter.memorySizeLimiter == nil && limiter.requestCountLimiter == nil { + return nextConsumer + } + return &metricsConsumer{ + nextConsumer: nextConsumer, + limiter: limiter, + } +} + +// WrapLogs wraps a logs consumer with resource limiters +func WrapLogs(provider extensionlimiter.Provider, nextConsumer consumer.Logs, options ...Option) consumer.Logs { + limiter := newLimiter(provider, options...) + if limiter.requestItemsLimiter == nil && limiter.memorySizeLimiter == nil && limiter.requestCountLimiter == nil { + return nextConsumer + } + return &logsConsumer{ + nextConsumer: nextConsumer, + limiter: limiter, + } +} + +// WrapProfiles wraps a profiles consumer with resource limiters +func WrapProfiles(provider extensionlimiter.Provider, nextConsumer xconsumer.Profiles, options ...Option) xconsumer.Profiles { + limiter := newLimiter(provider, options...) + + if limiter.requestItemsLimiter == nil && limiter.memorySizeLimiter == nil && limiter.requestCountLimiter == nil { + return nextConsumer + } + return &profilesConsumer{ + nextConsumer: nextConsumer, + limiter: limiter, + } +} + +// Signal-specific consumer implementations +type tracesConsumer struct { + nextConsumer consumer.Traces + *limiter +} + +func (tc *tracesConsumer) Capabilities() consumer.Capabilities { + return tc.nextConsumer.Capabilities() +} + +func (tc *tracesConsumer) ConsumeTraces(ctx context.Context, td ptrace.Traces) error { + numSpans := td.SpanCount() + if numSpans == 0 { + return tc.nextConsumer.ConsumeTraces(ctx, td) + } + + // Apply the request count limiter if available + if tc.requestCountLimiter != nil { + release, err := tc.requestCountLimiter.Acquire(ctx, 1) + defer release() + if err != nil { + return err + } + } + + // Apply the items limiter if available + if tc.requestItemsLimiter != nil { + release, err := tc.requestItemsLimiter.Acquire(ctx, uint64(numSpans)) + defer release() + if err != nil { + return err + } + } + + // Apply the memory size limiter if available + if tc.memorySizeLimiter != nil { + // Get the marshaled size of the request as a proxy for memory size + var sizer ptrace.ProtoMarshaler + size := sizer.TracesSize(td) + release, err := tc.memorySizeLimiter.Acquire(ctx, uint64(size)) + defer release() + if err != nil { + return err + } + } + + return tc.nextConsumer.ConsumeTraces(ctx, td) +} + +type metricsConsumer struct { + nextConsumer consumer.Metrics + *limiter +} + +func (mc *metricsConsumer) Capabilities() consumer.Capabilities { + return mc.nextConsumer.Capabilities() +} + +func (mc *metricsConsumer) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) error { + dataPointCount := md.DataPointCount() + if dataPointCount == 0 { + return mc.nextConsumer.ConsumeMetrics(ctx, md) + } + + // Apply the request count limiter if available + if mc.requestCountLimiter != nil { + release, err := mc.requestCountLimiter.Acquire(ctx, 1) + defer release() + if err != nil { + return err + } + } + + // Apply the items limiter if available + if mc.requestItemsLimiter != nil { + release, err := mc.requestItemsLimiter.Acquire(ctx, uint64(dataPointCount)) + defer release() + if err != nil { + return err + } + } + + // Apply the memory size limiter if available + if mc.memorySizeLimiter != nil { + var sizer pmetric.ProtoMarshaler + size := sizer.MetricsSize(md) + release, err := mc.memorySizeLimiter.Acquire(ctx, uint64(size)) + defer release() + if err != nil { + return err + } + } + + return mc.nextConsumer.ConsumeMetrics(ctx, md) +} + +type logsConsumer struct { + nextConsumer consumer.Logs + *limiter +} + +func (lc *logsConsumer) Capabilities() consumer.Capabilities { + return lc.nextConsumer.Capabilities() +} + +func (lc *logsConsumer) ConsumeLogs(ctx context.Context, ld plog.Logs) error { + numRecords := ld.LogRecordCount() + if numRecords == 0 { + return lc.nextConsumer.ConsumeLogs(ctx, ld) + } + + // Apply the request count limiter if available + if lc.requestCountLimiter != nil { + release, err := lc.requestCountLimiter.Acquire(ctx, 1) + defer release() + if err != nil { + return err + } + } + + // Apply the items limiter if available + if lc.requestItemsLimiter != nil { + release, err := lc.requestItemsLimiter.Acquire(ctx, uint64(numRecords)) + defer release() + if err != nil { + return err + } + } + + // Apply the memory size limiter if available + if lc.memorySizeLimiter != nil { + var sizer plog.ProtoMarshaler + size := sizer.LogsSize(ld) + release, err := lc.memorySizeLimiter.Acquire(ctx, uint64(size)) + defer release() + if err != nil { + return err + } + } + + return lc.nextConsumer.ConsumeLogs(ctx, ld) +} + +type profilesConsumer struct { + nextConsumer xconsumer.Profiles + *limiter +} + +func (pc *profilesConsumer) ConsumeProfiles(ctx context.Context, pd pprofile.Profiles) error { + numProfiles := pd.SampleCount() + if numProfiles == 0 { + return pc.nextConsumer.ConsumeProfiles(ctx, pd) + } + + // Apply the request count limiter if available + if pc.requestCountLimiter != nil { + release, err := pc.requestCountLimiter.Acquire(ctx, 1) + defer release() + if err != nil { + return err + } + } + + // Apply the items limiter if available + if pc.requestItemsLimiter != nil { + release, err := pc.requestItemsLimiter.Acquire(ctx, uint64(numProfiles)) + defer release() + if err != nil { + return err + } + } + + // Apply the memory size limiter if available + if pc.memorySizeLimiter != nil { + var sizer pprofile.ProtoMarshaler + size := sizer.ProfilesSize(pd) + release, err := pc.memorySizeLimiter.Acquire(ctx, uint64(size)) + defer release() + if err != nil { + return err + } + } + + return pc.nextConsumer.ConsumeProfiles(ctx, pd) +} + +func (pc *profilesConsumer) Capabilities() consumer.Capabilities { + return pc.nextConsumer.Capabilities() +} diff --git a/extension/extensionlimiter/limiterhelper/multi.go b/extension/extensionlimiter/limiterhelper/multi.go new file mode 100644 index 00000000000..bec3b0df3c6 --- /dev/null +++ b/extension/extensionlimiter/limiterhelper/multi.go @@ -0,0 +1,114 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limiterhelper // import "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configmiddleware" + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +// MiddlewaresToLimiterProvider returns a multi-provider that combines all middleware limiter providers. +func MiddlewaresToLimiterProvider(host component.Host, middlewares []configmiddleware.Config) extensionlimiter.Provider { + var providers []extensionlimiter.Provider + exts := host.GetExtensions() + for _, middleware := range middlewares { + ext := exts[middleware.ID] + if ext == nil { + continue + } + if provider, ok := ext.(extensionlimiter.Provider); ok { + providers = append(providers, provider) + } + } + return NewMultiProvider(providers...) +} + +// MultiProvider combines multiple Provider implementations into a single Provider. +// When requesting a limiter, it returns a combined limiter that applies all +// limits from the underlying providers. +type MultiProvider struct { + providers []extensionlimiter.Provider +} + +// NewMultiProvider creates a new MultiProvider from the given providers. +func NewMultiProvider(providers ...extensionlimiter.Provider) *MultiProvider { + return &MultiProvider{providers: providers} +} + +// RateLimiter returns a combined RateLimiter for the specified weight key. +// The combined limiter will apply all limits from the underlying providers. +func (mp *MultiProvider) RateLimiter(key extensionlimiter.WeightKey) extensionlimiter.RateLimiter { + limiters := make([]extensionlimiter.RateLimiter, 0, len(mp.providers)) + for _, p := range mp.providers { + if limiter := p.RateLimiter(key); limiter != nil { + limiters = append(limiters, limiter) + } + } + + if len(limiters) == 0 { + return nil + } + + return extensionlimiter.RateLimiterFunc(func(ctx context.Context, value uint64) error { + for _, limiter := range limiters { + if err := limiter.Limit(ctx, value); err != nil { + return err + } + } + return nil + }) +} + +// ResourceLimiter returns a combined ResourceLimiter for the specified weight key. +// The combined limiter will apply all limits from the underlying providers and +// aggregate the release functions. +func (mp *MultiProvider) ResourceLimiter(key extensionlimiter.WeightKey) extensionlimiter.ResourceLimiter { + limiters := make([]extensionlimiter.ResourceLimiter, 0, len(mp.providers)) + for _, p := range mp.providers { + if limiter := p.ResourceLimiter(key); limiter != nil { + limiters = append(limiters, limiter) + } + } + + if len(limiters) == 0 { + return nil + } + + return extensionlimiter.ResourceLimiterFunc(func(ctx context.Context, value uint64) (extensionlimiter.ReleaseFunc, error) { + var funcs releaseFuncs + + for _, limiter := range limiters { + releaseFunc, err := limiter.Acquire(ctx, value) + if err != nil { + // Release any already acquired resources + funcs.release() + return func() {}, err + } + if releaseFunc != nil { + funcs = append(funcs, releaseFunc) + } + } + + if len(funcs) == 0 { + return func() {}, nil + } + + return funcs.release, nil + }) +} + +// releaseFuncs is a collection of release functions that can be called together +type releaseFuncs []extensionlimiter.ReleaseFunc + +// release calls all non-nil release functions in the collection +func (r releaseFuncs) release() { + for _, rf := range r { + if rf != nil { + rf() + } + } +} diff --git a/extension/limitermiddlewareextension/Makefile b/extension/limitermiddlewareextension/Makefile new file mode 100644 index 00000000000..ded7a36092d --- /dev/null +++ b/extension/limitermiddlewareextension/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/extension/limitermiddlewareextension/config.go b/extension/limitermiddlewareextension/config.go new file mode 100644 index 00000000000..21f9774f93b --- /dev/null +++ b/extension/limitermiddlewareextension/config.go @@ -0,0 +1,27 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limitermiddlewareextension // import "go.opentelemetry.io/collector/extension/limitermiddlewareextension" + +import ( + "errors" + + "go.opentelemetry.io/collector/config/configlimiter" +) + +var errNoLimiterSpecified = errors.New("no limiter extension specified") + +// Config defines configuration for the limiter middleware extension. +type Config struct { + // Limiter configures the underlying extension used for limiting. + Limiter configlimiter.Config `mapstructure:"limiter,omitempty"` +} + +// Validate checks if the extension configuration is valid +func (cfg *Config) Validate() error { + var noID configlimiter.Config + if cfg.Limiter == noID { + return errNoLimiterSpecified + } + return nil +} diff --git a/extension/limitermiddlewareextension/factory.go b/extension/limitermiddlewareextension/factory.go new file mode 100644 index 00000000000..ea68cdf0cd3 --- /dev/null +++ b/extension/limitermiddlewareextension/factory.go @@ -0,0 +1,33 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limitermiddlewareextension // import "go.opentelemetry.io/collector/extension/limitermiddlewareextension" + +//go:generate mdatagen metadata.yaml + +import ( + "context" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + "go.opentelemetry.io/collector/extension/limitermiddlewareextension/internal/metadata" +) + +// NewFactory returns a new factory for the Limiter Middleware extension. +func NewFactory() extension.Factory { + return extension.NewFactory( + metadata.Type, + createDefaultConfig, + func(ctx context.Context, set extension.Settings, cfg component.Config) ( + extension.Extension, + error, + ) { + return newLimiterMiddleware(ctx, cfg.(*Config), set) + }, + metadata.ExtensionStability) +} + +// CreateDefaultConfig creates the default configuration for extension. +func createDefaultConfig() component.Config { + return &Config{} +} diff --git a/extension/limitermiddlewareextension/generated_component_test.go b/extension/limitermiddlewareextension/generated_component_test.go new file mode 100644 index 00000000000..9f3808b70c5 --- /dev/null +++ b/extension/limitermiddlewareextension/generated_component_test.go @@ -0,0 +1,42 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package limitermiddlewareextension + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/extension/extensiontest" +) + +var typ = component.MustNewType("limiter_middleware") + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, typ, NewFactory().Type()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + t.Run("shutdown", func(t *testing.T) { + e, err := factory.Create(context.Background(), extensiontest.NewNopSettings(typ), cfg) + require.NoError(t, err) + err = e.Shutdown(context.Background()) + require.NoError(t, err) + }) +} diff --git a/extension/limitermiddlewareextension/generated_package_test.go b/extension/limitermiddlewareextension/generated_package_test.go new file mode 100644 index 00000000000..0ee7137a0b8 --- /dev/null +++ b/extension/limitermiddlewareextension/generated_package_test.go @@ -0,0 +1,13 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package limitermiddlewareextension + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/extension/limitermiddlewareextension/go.mod b/extension/limitermiddlewareextension/go.mod new file mode 100644 index 00000000000..f531b29cdc1 --- /dev/null +++ b/extension/limitermiddlewareextension/go.mod @@ -0,0 +1,87 @@ +module go.opentelemetry.io/collector/extension/limitermiddlewareextension + +go 1.23.0 + +require ( + github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/collector/component v1.30.0 + go.opentelemetry.io/collector/component/componenttest v0.124.0 + go.opentelemetry.io/collector/config/configlimiter v0.0.0-00010101000000-000000000000 + go.opentelemetry.io/collector/confmap v1.30.0 + go.opentelemetry.io/collector/extension v1.30.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 + go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 + go.opentelemetry.io/collector/extension/extensiontest v0.124.0 + go.uber.org/goleak v1.3.0 + google.golang.org/grpc v1.71.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/knadh/koanf/providers/confmap v1.0.0 // indirect + github.com/knadh/koanf/v2 v2.2.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/featuregate v1.30.0 // indirect + go.opentelemetry.io/collector/internal/telemetry v0.124.0 // indirect + go.opentelemetry.io/collector/pdata v1.30.0 // indirect + go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/log v0.11.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/config/configlimiter => ../../config/configlimiter + +replace go.opentelemetry.io/collector/pdata => ../../pdata + +replace go.opentelemetry.io/collector/featuregate => ../../featuregate + +replace go.opentelemetry.io/collector/extension/extensiontest => ../extensiontest + +replace go.opentelemetry.io/collector/extension => ../ + +replace go.opentelemetry.io/collector/confmap => ../../confmap + +replace go.opentelemetry.io/collector/pipeline => ../../pipeline + +replace go.opentelemetry.io/collector/component/componenttest => ../../component/componenttest + +replace go.opentelemetry.io/collector/component => ../../component + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware + +replace go.opentelemetry.io/collector/consumer => ../../consumer + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extensionmiddleware/extensionmiddlewaretest diff --git a/extension/limitermiddlewareextension/go.sum b/extension/limitermiddlewareextension/go.sum new file mode 100644 index 00000000000..a2dcf9ccdb7 --- /dev/null +++ b/extension/limitermiddlewareextension/go.sum @@ -0,0 +1,110 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= +github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= +github.com/knadh/koanf/v2 v2.2.0 h1:FZFwd9bUjpb8DyCWARUBy5ovuhDs1lI87dOEn2K8UVU= +github.com/knadh/koanf/v2 v2.2.0/go.mod h1:PSFru3ufQgTsI7IF+95rf9s8XA1+aHxKuO/W+dPoHEY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y= +go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/extension/limitermiddlewareextension/internal/metadata/generated_status.go b/extension/limitermiddlewareextension/internal/metadata/generated_status.go new file mode 100644 index 00000000000..7b84ccfa498 --- /dev/null +++ b/extension/limitermiddlewareextension/internal/metadata/generated_status.go @@ -0,0 +1,16 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("limiter_middleware") + ScopeName = "go.opentelemetry.io/collector/extension/limitermiddlewareextension" +) + +const ( + ExtensionStability = component.StabilityLevelDevelopment +) diff --git a/extension/limitermiddlewareextension/limitermiddleware.go b/extension/limitermiddlewareextension/limitermiddleware.go new file mode 100644 index 00000000000..a7988339699 --- /dev/null +++ b/extension/limitermiddlewareextension/limitermiddleware.go @@ -0,0 +1,356 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package limitermiddlewareextension // import "go.opentelemetry.io/collector/extension/limitermiddlewareextension" + +import ( + "context" + "io" + "net/http" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/stats" + "google.golang.org/grpc/status" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config/configlimiter" + "go.opentelemetry.io/collector/extension" + "go.opentelemetry.io/collector/extension/extensionlimiter" + "go.opentelemetry.io/collector/extension/extensionmiddleware" +) + +// tooManyRequestsMsg is the standard text for 429 status code +var tooManyRequestsMsg = http.StatusText(http.StatusTooManyRequests) + +// limiterMiddleware implements rate limiting across various transports. +type limiterMiddleware struct { + id configlimiter.Config + provider extensionlimiter.Provider +} + +// newLimiterMiddleware creates a new limiter middleware instance. +func newLimiterMiddleware(_ context.Context, cfg *Config, _ extension.Settings) (*limiterMiddleware, error) { + return &limiterMiddleware{ + id: cfg.Limiter, + }, nil +} + +// Ensure limiterMiddleware implements all required middleware interfaces. +var ( + _ extensionmiddleware.HTTPClient = &limiterMiddleware{} + _ extensionmiddleware.HTTPServer = &limiterMiddleware{} + _ extensionmiddleware.GRPCClient = &limiterMiddleware{} + _ extensionmiddleware.GRPCServer = &limiterMiddleware{} + _ component.Component = &limiterMiddleware{} +) + +// Start initializes the limiter by getting it from host extensions. +func (lm *limiterMiddleware) Start(ctx context.Context, host component.Host) error { + provider, err := lm.id.GetProvider(ctx, host.GetExtensions()) + if err != nil { + return err + } + lm.provider = provider + return nil +} + +// Shutdown cleans up resources used by the limiter middleware. +func (lm *limiterMiddleware) Shutdown(_ context.Context) error { + return nil +} + +type limiterRoundTripper struct { + base http.RoundTripper + resourceLimiter extensionlimiter.ResourceLimiter + rateLimiter extensionlimiter.RateLimiter +} + +// limitExceeded creates an HTTP 429 (Too Many Requests) response from the client. +func limitExceeded(req *http.Request) *http.Response { + return &http.Response{ + StatusCode: http.StatusTooManyRequests, + Status: tooManyRequestsMsg, + Request: req, + ContentLength: 0, + Body: http.NoBody, + } +} + +// GetHTTPRoundTripper returns an HTTP roundtripper that applies rate limiting. +func (lm *limiterMiddleware) GetHTTPRoundTripper(base http.RoundTripper) (http.RoundTripper, error) { + resourceLimiter := lm.provider.ResourceLimiter(extensionlimiter.WeightKeyRequestCount) + rateLimiter := lm.provider.RateLimiter(extensionlimiter.WeightKeyNetworkBytes) + + if resourceLimiter == nil && rateLimiter == nil { + // If no limiters are configured, return the base round tripper + return base, nil + } + + return &limiterRoundTripper{ + base: base, + resourceLimiter: resourceLimiter, + rateLimiter: rateLimiter, + }, nil +} + +// rateLimitedBody wraps an http.Request.Body to track bytes and call the rate limiter +type rateLimitedBody struct { + body io.ReadCloser + rateLimiter extensionlimiter.RateLimiter + ctx context.Context +} + +// Read implements io.Reader interface, counting bytes as they are read +func (rb *rateLimitedBody) Read(p []byte) (n int, err error) { + n, err = rb.body.Read(p) + if n > 0 { + // Apply rate limiting based on network bytes after they are read + limitErr := rb.rateLimiter.Limit(rb.ctx, uint64(n)) + if limitErr != nil { + // If the rate limiter rejects the bytes, return the error + return n, limitErr + } + } + return n, err +} + +// Close implements io.Closer interface +func (rb *rateLimitedBody) Close() error { + return rb.body.Close() +} + +// RoundTrip implements http.RoundTripper with rate limiting. +func (lrt *limiterRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // Apply resource limit check for request count + if lrt.resourceLimiter != nil { + release, err := lrt.resourceLimiter.Acquire(req.Context(), 1) + defer release() + if err != nil { + return limitExceeded(req), nil + } + } + + // Create a new request with a body that tracks network bytes + newReq := req.Clone(req.Context()) + if lrt.rateLimiter != nil && req.Body != nil && req.Body != http.NoBody { + newReq.Body = &rateLimitedBody{ + body: req.Body, + rateLimiter: lrt.rateLimiter, + ctx: req.Context(), + } + } + + return lrt.base.RoundTrip(newReq) +} + +// GetHTTPHandler wraps an HTTP handler with rate limiting. +func (lm *limiterMiddleware) GetHTTPHandler(base http.Handler) (http.Handler, error) { + resourceLimiter := lm.provider.ResourceLimiter(extensionlimiter.WeightKeyRequestCount) + rateLimiter := lm.provider.RateLimiter(extensionlimiter.WeightKeyNetworkBytes) + + if resourceLimiter == nil && rateLimiter == nil { + return base, nil + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Apply resource limit check for request count + if resourceLimiter != nil { + release, err := resourceLimiter.Acquire(r.Context(), 1) + defer release() + if err != nil { + http.Error(w, tooManyRequestsMsg, http.StatusTooManyRequests) + return + } + } + + if rateLimiter != nil { + // Create a new request with a body that tracks network bytes + newReq := r.Clone(r.Context()) + if r.Body != nil && r.Body != http.NoBody { + newReq.Body = &rateLimitedBody{ + body: r.Body, + rateLimiter: rateLimiter, + ctx: r.Context(), + } + } + r = newReq + } + + base.ServeHTTP(w, r) + }), nil +} + +func (lm *limiterMiddleware) GetGRPCClientOptions() (options []grpc.DialOption, _ error) { + if resourceLimiter := lm.provider.ResourceLimiter(extensionlimiter.WeightKeyRequestCount); resourceLimiter != nil { + options = append(options, grpc.WithUnaryInterceptor( + func( + ctx context.Context, + method string, + req, reply any, + cc *grpc.ClientConn, + invoker grpc.UnaryInvoker, + opts ...grpc.CallOption, + ) error { + release, err := resourceLimiter.Acquire(ctx, 1) + defer release() + if err != nil { + return status.Errorf(codes.ResourceExhausted, "limit exceeded: %v", err) + } + return invoker(ctx, method, req, reply, cc, opts...) + }), + grpc.WithStreamInterceptor( + func( + ctx context.Context, + desc *grpc.StreamDesc, + cc *grpc.ClientConn, + method string, + streamer grpc.Streamer, + opts ...grpc.CallOption, + ) (grpc.ClientStream, error) { + cstream, err := streamer(ctx, desc, cc, method, opts...) + if err != nil { + return nil, err + } + return lm.wrapClientStream(cstream, method, resourceLimiter), nil + }), + ) + } + if rateLimiter := lm.provider.RateLimiter(extensionlimiter.WeightKeyNetworkBytes); rateLimiter != nil { + options = append(options, grpc.WithStatsHandler( + &limiterStatsHandler{ + rateLimiter: rateLimiter, + isClient: true, + })) + } + return options, nil +} + +// ServerUnaryInterceptor returns a gRPC interceptor for unary server calls. +func (lm *limiterMiddleware) GetGRPCServerOptions() (options []grpc.ServerOption, _ error) { + if resourceLimiter := lm.provider.ResourceLimiter(extensionlimiter.WeightKeyRequestCount); resourceLimiter != nil { + options = append(options, grpc.ChainUnaryInterceptor( + // The unary resource case + func( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + ) (any, error) { + release, err := resourceLimiter.Acquire(ctx, 1) + defer release() + if err != nil { + return nil, status.Errorf(codes.ResourceExhausted, "limit exceeded: %v", err) + } + return handler(ctx, req) + }), grpc.ChainStreamInterceptor( + // The stream resource case + func( + srv interface{}, + ss grpc.ServerStream, + info *grpc.StreamServerInfo, + handler grpc.StreamHandler, + ) error { + return handler(srv, lm.wrapServerStream(ss, info, resourceLimiter)) + }), + ) + } + if rateLimiter := lm.provider.RateLimiter(extensionlimiter.WeightKeyNetworkBytes); rateLimiter != nil { + options = append(options, grpc.StatsHandler( + &limiterStatsHandler{ + rateLimiter: rateLimiter, + isClient: false, + })) + } + + return options, nil +} + +// limiterStatsHandler implements the stats.Handler interface for rate limiting. +type limiterStatsHandler struct { + rateLimiter extensionlimiter.RateLimiter + isClient bool +} + +func (h *limiterStatsHandler) TagRPC(ctx context.Context, _ *stats.RPCTagInfo) context.Context { + return ctx +} + +func (h *limiterStatsHandler) HandleRPC(ctx context.Context, s stats.RPCStats) { + // Check for payload messages to apply network byte rate limiting + var wireBytes int + switch payload := s.(type) { + case *stats.InPayload: + // Server receiving payload (or client receiving response) + if !h.isClient { + wireBytes = payload.WireLength + } + case *stats.OutPayload: + // Client sending payload (or server sending response) + if h.isClient { + wireBytes = payload.WireLength + } + default: + // Not a payload message, no rate limiting to apply + return + } + + if wireBytes == 0 { + return + } + // Apply rate limiting based on network bytes + h.rateLimiter.Limit(ctx, uint64(wireBytes)) +} + +func (h *limiterStatsHandler) TagConn(ctx context.Context, _ *stats.ConnTagInfo) context.Context { + return ctx +} + +func (h *limiterStatsHandler) HandleConn(ctx context.Context, _ stats.ConnStats) { +} + +type serverStream struct { + grpc.ServerStream + limiter extensionlimiter.ResourceLimiter +} + +// RecvMsg applies rate limiting to server stream message receiving. +func (s *serverStream) RecvMsg(m any) error { + release, err := s.limiter.Acquire(s.Context(), 1) + defer release() + if err != nil { + return status.Errorf(codes.ResourceExhausted, "limit exceeded: %v", err) + } + return s.ServerStream.RecvMsg(m) +} + +// wrapServerStream wraps a gRPC server stream with rate limiting. +func (lm *limiterMiddleware) wrapServerStream(ss grpc.ServerStream, _ *grpc.StreamServerInfo, limiter extensionlimiter.ResourceLimiter) grpc.ServerStream { + return &serverStream{ + ServerStream: ss, + limiter: limiter, + } +} + +type clientStream struct { + grpc.ClientStream + limiter extensionlimiter.ResourceLimiter +} + +// SendMsg applies rate limiting to client stream message sending. +func (s *clientStream) SendMsg(m any) error { + release, err := s.limiter.Acquire(s.Context(), 1) + defer release() + if err != nil { + return status.Errorf(codes.ResourceExhausted, "limit exceeded: %v", err) + } + return s.ClientStream.SendMsg(m) +} + +// wrapClientStream wraps a gRPC client stream with rate limiting. +func (lm *limiterMiddleware) wrapClientStream(cs grpc.ClientStream, _ string, limiter extensionlimiter.ResourceLimiter) grpc.ClientStream { + return &clientStream{ + ClientStream: cs, + limiter: limiter, + } +} diff --git a/extension/limitermiddlewareextension/metadata.yaml b/extension/limitermiddlewareextension/metadata.yaml new file mode 100644 index 00000000000..73c519da843 --- /dev/null +++ b/extension/limitermiddlewareextension/metadata.yaml @@ -0,0 +1,12 @@ +type: limiter_middleware +github_project: open-telemetry/opentelemetry-collector + +status: + class: extension + stability: + development: [extension] + distributions: [] + +tests: + # note: not sure how to configure the associated limiter extension + skip_lifecycle: true diff --git a/extension/memorylimiterextension/go.mod b/extension/memorylimiterextension/go.mod index 8fdb6425847..c0da0092ad1 100644 --- a/extension/memorylimiterextension/go.mod +++ b/extension/memorylimiterextension/go.mod @@ -8,6 +8,7 @@ require ( go.opentelemetry.io/collector/component/componenttest v0.124.0 go.opentelemetry.io/collector/confmap v1.30.0 go.opentelemetry.io/collector/extension v1.30.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 go.opentelemetry.io/collector/extension/extensiontest v0.124.0 go.opentelemetry.io/collector/internal/memorylimiter v0.124.0 go.uber.org/goleak v1.3.0 @@ -66,6 +67,8 @@ replace go.opentelemetry.io/collector/confmap => ../../confmap replace go.opentelemetry.io/collector/extension => ../../extension +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../extensionlimiter + replace go.opentelemetry.io/collector/pdata => ../../pdata replace go.opentelemetry.io/collector/internal/memorylimiter => ../../internal/memorylimiter @@ -77,3 +80,15 @@ replace go.opentelemetry.io/collector/featuregate => ../../featuregate replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry replace go.opentelemetry.io/collector/pipeline => ../../pipeline + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../extensionmiddleware + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/consumer => ../../consumer + +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extensionmiddleware/extensionmiddlewaretest diff --git a/extension/memorylimiterextension/memorylimiter.go b/extension/memorylimiterextension/memorylimiter.go index 4986243ce10..63af2e718a5 100644 --- a/extension/memorylimiterextension/memorylimiter.go +++ b/extension/memorylimiterextension/memorylimiter.go @@ -5,10 +5,12 @@ package memorylimiterextension // import "go.opentelemetry.io/collector/extensio import ( "context" + "errors" "go.uber.org/zap" "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension/extensionlimiter" "go.opentelemetry.io/collector/internal/memorylimiter" ) @@ -16,6 +18,10 @@ type memoryLimiterExtension struct { memLimiter *memorylimiter.MemoryLimiter } +var _ extensionlimiter.Provider = &memoryLimiterExtension{} + +var ErrMemoryLimitExceeded = errors.New("memory limit exceeded") + // newMemoryLimiter returns a new memorylimiter extension. func newMemoryLimiter(cfg *Config, logger *zap.Logger) (*memoryLimiterExtension, error) { ml, err := memorylimiter.NewMemoryLimiter(cfg, logger) @@ -26,6 +32,26 @@ func newMemoryLimiter(cfg *Config, logger *zap.Logger) (*memoryLimiterExtension, return &memoryLimiterExtension{memLimiter: ml}, nil } +// RateLimiter implements extensionlimiter.Provider. +func (ml *memoryLimiterExtension) RateLimiter(_ extensionlimiter.WeightKey) extensionlimiter.RateLimiter { + return extensionlimiter.RateLimiterFunc(func(_ context.Context, _ uint64) error { + if ml.memLimiter.MustRefuse() { + return ErrMemoryLimitExceeded + } + return nil + }) +} + +// ResourceLimiter implements extensionlimiter.Provider. +func (ml *memoryLimiterExtension) ResourceLimiter(key extensionlimiter.WeightKey) extensionlimiter.ResourceLimiter { + return extensionlimiter.ResourceLimiterFunc(func(_ context.Context, _ uint64) (extensionlimiter.ReleaseFunc, error) { + if ml.memLimiter.MustRefuse() { + return func() {}, ErrMemoryLimitExceeded + } + return func() {}, nil + }) +} + func (ml *memoryLimiterExtension) Start(ctx context.Context, host component.Host) error { return ml.memLimiter.Start(ctx, host) } @@ -34,7 +60,9 @@ func (ml *memoryLimiterExtension) Shutdown(ctx context.Context) error { return ml.memLimiter.Shutdown(ctx) } -// MustRefuse returns if the caller should deny because memory has reached it's configured limits +// MustRefuse returns if the caller should deny because memory has +// reached its configured limits. This is the original API; it is not +// clear that there are any callers other than memorylimiterprocessor. func (ml *memoryLimiterExtension) MustRefuse() bool { return ml.memLimiter.MustRefuse() } diff --git a/extension/ratelimiterextension/Makefile b/extension/ratelimiterextension/Makefile new file mode 100644 index 00000000000..ded7a36092d --- /dev/null +++ b/extension/ratelimiterextension/Makefile @@ -0,0 +1 @@ +include ../../Makefile.Common diff --git a/extension/ratelimiterextension/config.go b/extension/ratelimiterextension/config.go new file mode 100644 index 00000000000..717047dffe3 --- /dev/null +++ b/extension/ratelimiterextension/config.go @@ -0,0 +1,59 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ratelimiterextension // import "go.opentelemetry.io/collector/extension/ratelimiterextension" + +import ( + "errors" + "fmt" + "time" + + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +// LimitConfig defines parameters for a token bucket rate limiter +type LimitConfig struct { + // Key specifies the weight key to limit (e.g. "network_bytes", "request_count", etc) + Key extensionlimiter.WeightKey `mapstructure:"key"` + // Rate is the rate at which tokens are replenished (e.g. X tokens per second) + Rate float64 `mapstructure:"rate"` + // BurstSize is the maximum number of tokens that can be consumed in a single burst + BurstSize int64 `mapstructure:"burst_size"` +} + +// Config defines the configuration for the token bucket rate limiter extension +type Config struct { + // Limits is a list of token bucket configurations, one per weight key + Limits []LimitConfig `mapstructure:"limits"` + // FillingInterval is the interval at which tokens are added to the bucket + FillingInterval time.Duration `mapstructure:"filling_interval"` +} + +// Validate checks if the extension configuration is valid +func (cfg *Config) Validate() error { + if len(cfg.Limits) == 0 { + return errors.New("at least one limit must be configured") + } + if cfg.FillingInterval <= 0 { + return fmt.Errorf("filling_interval must be positive, got %v", cfg.FillingInterval) + } + + seenKeys := make(map[extensionlimiter.WeightKey]bool) + for i, limit := range cfg.Limits { + if limit.Key == "" { + return fmt.Errorf("limit[%d].key cannot be empty", i) + } + if seenKeys[limit.Key] { + return fmt.Errorf("duplicate limit key: %s", limit.Key) + } + seenKeys[limit.Key] = true + + if limit.Rate <= 0 { + return fmt.Errorf("limit[%d].rate must be positive, got %v", i, limit.Rate) + } + if limit.BurstSize <= 0 { + return fmt.Errorf("limit[%d].burst_size must be positive, got %v", i, limit.BurstSize) + } + } + return nil +} diff --git a/extension/ratelimiterextension/factory.go b/extension/ratelimiterextension/factory.go new file mode 100644 index 00000000000..dab8bbf3cb1 --- /dev/null +++ b/extension/ratelimiterextension/factory.go @@ -0,0 +1,45 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ratelimiterextension // import "go.opentelemetry.io/collector/extension/ratelimiterextension" + +import ( + "context" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + "go.opentelemetry.io/collector/extension/extensionlimiter" + "go.opentelemetry.io/collector/extension/ratelimiterextension/internal/metadata" +) + +//go:generate mdatagen metadata.yaml + +// NewFactory creates a factory for the rate limiter extension. +func NewFactory() extension.Factory { + return extension.NewFactory( + metadata.Type, + createDefaultConfig, + func(ctx context.Context, set extension.Settings, cfg component.Config) ( + extension.Extension, + error, + ) { + return newRateLimiter(ctx, set, cfg.(*Config)) + }, + metadata.ExtensionStability, + ) +} + +// createDefaultConfig creates a relatively large default config. +func createDefaultConfig() component.Config { + return &Config{ + Limits: []LimitConfig{ + { + Key: extensionlimiter.WeightKeyNetworkBytes, + Rate: 10485760.0, // 10 MiB per second + BurstSize: 52428800, // Allow bursts of up to 50 MiB + }, + }, + FillingInterval: 100 * time.Millisecond, // Check for tokens every 100ms + } +} diff --git a/extension/ratelimiterextension/generated_component_test.go b/extension/ratelimiterextension/generated_component_test.go new file mode 100644 index 00000000000..ca59e723697 --- /dev/null +++ b/extension/ratelimiterextension/generated_component_test.go @@ -0,0 +1,42 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package ratelimiterextension + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/confmap/confmaptest" + "go.opentelemetry.io/collector/extension/extensiontest" +) + +var typ = component.MustNewType("rate_limiter") + +func TestComponentFactoryType(t *testing.T) { + require.Equal(t, typ, NewFactory().Type()) +} + +func TestComponentConfigStruct(t *testing.T) { + require.NoError(t, componenttest.CheckConfigStruct(NewFactory().CreateDefaultConfig())) +} + +func TestComponentLifecycle(t *testing.T) { + factory := NewFactory() + + cm, err := confmaptest.LoadConf("metadata.yaml") + require.NoError(t, err) + cfg := factory.CreateDefaultConfig() + sub, err := cm.Sub("tests::config") + require.NoError(t, err) + require.NoError(t, sub.Unmarshal(&cfg)) + t.Run("shutdown", func(t *testing.T) { + e, err := factory.Create(context.Background(), extensiontest.NewNopSettings(typ), cfg) + require.NoError(t, err) + err = e.Shutdown(context.Background()) + require.NoError(t, err) + }) +} diff --git a/extension/ratelimiterextension/generated_package_test.go b/extension/ratelimiterextension/generated_package_test.go new file mode 100644 index 00000000000..208f1e4df4d --- /dev/null +++ b/extension/ratelimiterextension/generated_package_test.go @@ -0,0 +1,13 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package ratelimiterextension + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/extension/ratelimiterextension/go.mod b/extension/ratelimiterextension/go.mod new file mode 100644 index 00000000000..ae76a14021b --- /dev/null +++ b/extension/ratelimiterextension/go.mod @@ -0,0 +1,83 @@ +module go.opentelemetry.io/collector/extension/ratelimiterextension + +go 1.23.0 + +require ( + github.com/stretchr/testify v1.10.0 + go.opentelemetry.io/collector/component v1.30.0 + go.opentelemetry.io/collector/component/componenttest v0.124.0 + go.opentelemetry.io/collector/confmap v1.28.1 + go.opentelemetry.io/collector/extension v1.30.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 + go.opentelemetry.io/collector/extension/extensiontest v0.122.1 + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/knadh/koanf/maps v0.1.2 // indirect + github.com/knadh/koanf/providers/confmap v1.0.0 // indirect + github.com/knadh/koanf/v2 v2.2.0 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/collector/featuregate v1.30.0 // indirect + go.opentelemetry.io/collector/internal/telemetry v0.124.0 // indirect + go.opentelemetry.io/collector/pdata v1.30.0 // indirect + go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/log v0.11.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/grpc v1.71.1 // indirect + google.golang.org/protobuf v1.36.6 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + +replace go.opentelemetry.io/collector/component => ../../component + +replace go.opentelemetry.io/collector/internal/telemetry => ../../internal/telemetry + +replace go.opentelemetry.io/collector/extension => ../ + +replace go.opentelemetry.io/collector/featuregate => ../../featuregate + +replace go.opentelemetry.io/collector/extension/extensiontest => ../extensiontest + +replace go.opentelemetry.io/collector/confmap => ../../confmap + +replace go.opentelemetry.io/collector/component/componenttest => ../../component/componenttest + +replace go.opentelemetry.io/collector/pdata => ../../pdata + +replace go.opentelemetry.io/collector/pipeline => ../../pipeline + +replace go.opentelemetry.io/collector/consumer/xconsumer => ../../consumer/xconsumer + +replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../extensionmiddleware + +replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware + +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/pdata/pprofile => ../../pdata/pprofile + +replace go.opentelemetry.io/collector/consumer => ../../consumer diff --git a/extension/ratelimiterextension/go.sum b/extension/ratelimiterextension/go.sum new file mode 100644 index 00000000000..a2dcf9ccdb7 --- /dev/null +++ b/extension/ratelimiterextension/go.sum @@ -0,0 +1,110 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= +github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE= +github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= +github.com/knadh/koanf/v2 v2.2.0 h1:FZFwd9bUjpb8DyCWARUBy5ovuhDs1lI87dOEn2K8UVU= +github.com/knadh/koanf/v2 v2.2.0/go.mod h1:PSFru3ufQgTsI7IF+95rf9s8XA1+aHxKuO/W+dPoHEY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0 h1:ojdSRDvjrnm30beHOmwsSvLpoRF40MlwNCA+Oo93kXU= +go.opentelemetry.io/contrib/bridges/otelzap v0.10.0/go.mod h1:oTTm4g7NEtHSV2i/0FeVdPaPgUIZPfQkFbq0vbzqnv0= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/log v0.11.0 h1:c24Hrlk5WJ8JWcwbQxdBqxZdOK7PcP/LFtOtwpDTe3Y= +go.opentelemetry.io/otel/log v0.11.0/go.mod h1:U/sxQ83FPmT29trrifhQg+Zj2lo1/IPN1PF6RTFqdwc= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/extension/ratelimiterextension/internal/metadata/generated_status.go b/extension/ratelimiterextension/internal/metadata/generated_status.go new file mode 100644 index 00000000000..cf6d2dbd757 --- /dev/null +++ b/extension/ratelimiterextension/internal/metadata/generated_status.go @@ -0,0 +1,16 @@ +// Code generated by mdatagen. DO NOT EDIT. + +package metadata + +import ( + "go.opentelemetry.io/collector/component" +) + +var ( + Type = component.MustNewType("rate_limiter") + ScopeName = "go.opentelemetry.io/collector/extension/ratelimiterextension" +) + +const ( + ExtensionStability = component.StabilityLevelDevelopment +) diff --git a/extension/ratelimiterextension/metadata.yaml b/extension/ratelimiterextension/metadata.yaml new file mode 100644 index 00000000000..d867690a2e6 --- /dev/null +++ b/extension/ratelimiterextension/metadata.yaml @@ -0,0 +1,12 @@ +type: rate_limiter +github_project: open-telemetry/opentelemetry-collector + +status: + class: extension + stability: + development: [extension] + distributions: [] + +tests: + # note: not sure how to configure the associated limiter extension + skip_lifecycle: true diff --git a/extension/ratelimiterextension/ratelimiter.go b/extension/ratelimiterextension/ratelimiter.go new file mode 100644 index 00000000000..024fd234361 --- /dev/null +++ b/extension/ratelimiterextension/ratelimiter.go @@ -0,0 +1,160 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package ratelimiterextension // import "go.opentelemetry.io/collector/extension/ratelimiterextension" + +import ( + "context" + "sync" + "time" + + "go.uber.org/zap" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/extension" + "go.opentelemetry.io/collector/extension/extensionlimiter" +) + +// tokenBucket implements a token bucket rate limiter +type tokenBucket struct { + // rate at which tokens are added to the bucket per second + rate float64 + // maximum size of the bucket + burstSize int64 + // current number of tokens in the bucket + tokens int64 + // last time tokens were added to the bucket + lastUpdate time.Time + // mutex to protect the token bucket + mu sync.Mutex +} + +// newTokenBucket creates a new token bucket with the specified rate and burst size +func newTokenBucket(rate float64, burstSize int64) *tokenBucket { + return &tokenBucket{ + rate: rate, + burstSize: burstSize, + tokens: burstSize, // Start with a full bucket + lastUpdate: time.Now(), + } +} + +// updateTokens adds tokens to the bucket based on elapsed time +func (tb *tokenBucket) updateTokens() { + now := time.Now() + elapsed := now.Sub(tb.lastUpdate).Seconds() + + // Calculate tokens to add based on elapsed time and rate + newTokens := int64(elapsed * tb.rate) + + if newTokens > 0 { + tb.tokens += newTokens + if tb.tokens > tb.burstSize { + tb.tokens = tb.burstSize // Cap at burst size + } + tb.lastUpdate = now + } +} + +// tryConsume attempts to consume tokens and returns whether it succeeded +// If it didn't succeed, it returns the wait time needed +func (tb *tokenBucket) tryConsume(count int64, interval time.Duration) (bool, time.Duration) { + tb.mu.Lock() + defer tb.mu.Unlock() + + tb.updateTokens() + + if tb.tokens >= count { + tb.tokens -= count + return true, 0 + } + + // Calculate the time needed to get enough tokens + neededTokens := count - tb.tokens + waitTime := time.Duration(float64(neededTokens) / tb.rate * float64(time.Second)) + + // Limit wait time to the filling interval to avoid long waits + if waitTime > interval { + waitTime = interval + } + + return false, waitTime +} + +// consume blocks until the specified number of tokens can be consumed +func (tb *tokenBucket) consume(ctx context.Context, count int64, interval time.Duration) error { + for { + consumed, waitTime := tb.tryConsume(count, interval) + if consumed { + return nil + } + + // Wait for more tokens or until context is canceled + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(waitTime): + // Continue and try again + } + } +} + +// rateLimiterExtension implements a token bucket rate limiter and the Provider interface +type rateLimiterExtension struct { + limiters map[extensionlimiter.WeightKey]*tokenBucket + fillingInterval time.Duration + logger *zap.Logger +} + +// Ensure rateLimiterExtension implements the Provider interface +var _ extensionlimiter.Provider = (*rateLimiterExtension)(nil) + +// newRateLimiter creates a new rate limiter extension with the given config +func newRateLimiter(_ context.Context, settings extension.Settings, cfg *Config) (*rateLimiterExtension, error) { + limiters := make(map[extensionlimiter.WeightKey]*tokenBucket) + + for _, limit := range cfg.Limits { + limiters[limit.Key] = newTokenBucket(limit.Rate, limit.BurstSize) + } + + return &rateLimiterExtension{ + limiters: limiters, + fillingInterval: cfg.FillingInterval, + logger: settings.Logger, + }, nil +} + +// Start initializes the extension (no-op for this extension) +func (rl *rateLimiterExtension) Start(ctx context.Context, host component.Host) error { + return nil +} + +// Shutdown stops the extension (no-op for this extension) +func (rl *rateLimiterExtension) Shutdown(ctx context.Context) error { + return nil +} + +// RateLimiter returns a RateLimiter for the specified weight key or nil if not configured +func (rl *rateLimiterExtension) RateLimiter(key extensionlimiter.WeightKey) extensionlimiter.RateLimiter { + limiter, ok := rl.limiters[key] + if !ok { + // No limiter configured for this weight key + return nil + } + + // Return a RateLimiterFunc that adapts our tokenBucket to the RateLimiter interface + return extensionlimiter.RateLimiterFunc(func(ctx context.Context, weight uint64) error { + return limiter.consume(ctx, int64(weight), rl.fillingInterval) + }) +} + +// ResourceLimiter returns a ResourceLimiter for the specified weight key +func (rl *rateLimiterExtension) ResourceLimiter(key extensionlimiter.WeightKey) extensionlimiter.ResourceLimiter { + rate := rl.RateLimiter(key) + if rate == nil { + return nil + } + return extensionlimiter.ResourceLimiterFunc(func(ctx context.Context, weight uint64) (extensionlimiter.ReleaseFunc, error) { + return func() {}, rate.Limit(ctx, weight) + }) +} diff --git a/internal/e2e/go.mod b/internal/e2e/go.mod index 006d2ff67b9..3a90af4ae4e 100644 --- a/internal/e2e/go.mod +++ b/internal/e2e/go.mod @@ -92,6 +92,7 @@ require ( go.opentelemetry.io/collector/exporter/xexporter v0.124.0 // indirect go.opentelemetry.io/collector/extension/extensionauth v1.30.0 // indirect go.opentelemetry.io/collector/extension/extensioncapabilities v0.124.0 // indirect + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 // indirect go.opentelemetry.io/collector/extension/extensionmiddleware v1.30.0 // indirect go.opentelemetry.io/collector/extension/extensiontest v0.124.0 // indirect go.opentelemetry.io/collector/extension/xextension v0.124.0 // indirect @@ -264,8 +265,10 @@ replace go.opentelemetry.io/collector/confmap/provider/fileprovider => ../../con replace go.opentelemetry.io/collector/service/hostcapabilities => ../../service/hostcapabilities -replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest +replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware -replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware +replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest + +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter diff --git a/receiver/otlpreceiver/go.mod b/receiver/otlpreceiver/go.mod index f4538d79c44..b319e461801 100644 --- a/receiver/otlpreceiver/go.mod +++ b/receiver/otlpreceiver/go.mod @@ -22,6 +22,7 @@ require ( go.opentelemetry.io/collector/consumer/consumererror v0.124.0 go.opentelemetry.io/collector/consumer/consumertest v0.124.0 go.opentelemetry.io/collector/consumer/xconsumer v0.124.0 + go.opentelemetry.io/collector/extension/extensionlimiter v0.0.0-00010101000000-000000000000 go.opentelemetry.io/collector/internal/sharedcomponent v0.124.0 go.opentelemetry.io/collector/internal/telemetry v0.124.0 go.opentelemetry.io/collector/pdata v1.30.0 @@ -156,6 +157,8 @@ replace go.opentelemetry.io/collector/extension/extensionauth/extensionauthtest replace go.opentelemetry.io/collector/extension/extensionmiddleware => ../../extension/extensionmiddleware +replace go.opentelemetry.io/collector/extension/extensionlimiter => ../../extension/extensionlimiter + replace go.opentelemetry.io/collector/config/configmiddleware => ../../config/configmiddleware replace go.opentelemetry.io/collector/extension/extensionmiddleware/extensionmiddlewaretest => ../../extension/extensionmiddleware/extensionmiddlewaretest diff --git a/receiver/otlpreceiver/otlp.go b/receiver/otlpreceiver/otlp.go index 7e5ab5d4b30..b76b945c2aa 100644 --- a/receiver/otlpreceiver/otlp.go +++ b/receiver/otlpreceiver/otlp.go @@ -18,6 +18,7 @@ import ( "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/consumer/xconsumer" + "go.opentelemetry.io/collector/extension/extensionlimiter/limiterhelper" "go.opentelemetry.io/collector/internal/telemetry" "go.opentelemetry.io/collector/internal/telemetry/componentattribute" "go.opentelemetry.io/collector/pdata/plog/plogotlp" @@ -97,20 +98,26 @@ func (r *otlpReceiver) startGRPCServer(host component.Host) error { return err } + limiterProvider := limiterhelper.MiddlewaresToLimiterProvider(host, r.cfg.GRPC.Middlewares) + helperOptions := []limiterhelper.Option{ + limiterhelper.WithRequestItemsLimit(), + limiterhelper.WithMemorySizeLimit(), + } + if r.nextTraces != nil { - ptraceotlp.RegisterGRPCServer(r.serverGRPC, trace.New(r.nextTraces, r.obsrepGRPC)) + ptraceotlp.RegisterGRPCServer(r.serverGRPC, trace.New(limiterhelper.WrapTraces(limiterProvider, r.nextTraces, helperOptions...), r.obsrepGRPC)) } if r.nextMetrics != nil { - pmetricotlp.RegisterGRPCServer(r.serverGRPC, metrics.New(r.nextMetrics, r.obsrepGRPC)) + pmetricotlp.RegisterGRPCServer(r.serverGRPC, metrics.New(limiterhelper.WrapMetrics(limiterProvider, r.nextMetrics, helperOptions...), r.obsrepGRPC)) } if r.nextLogs != nil { - plogotlp.RegisterGRPCServer(r.serverGRPC, logs.New(r.nextLogs, r.obsrepGRPC)) + plogotlp.RegisterGRPCServer(r.serverGRPC, logs.New(limiterhelper.WrapLogs(limiterProvider, r.nextLogs, helperOptions...), r.obsrepGRPC)) } if r.nextProfiles != nil { - pprofileotlp.RegisterGRPCServer(r.serverGRPC, profiles.New(r.nextProfiles)) + pprofileotlp.RegisterGRPCServer(r.serverGRPC, profiles.New(limiterhelper.WrapProfiles(limiterProvider, r.nextProfiles, helperOptions...))) } r.settings.Logger.Info("Starting GRPC server", zap.String("endpoint", r.cfg.GRPC.NetAddr.Endpoint)) @@ -136,30 +143,36 @@ func (r *otlpReceiver) startHTTPServer(ctx context.Context, host component.Host) return nil } + limiterProvider := limiterhelper.MiddlewaresToLimiterProvider(host, r.cfg.HTTP.ServerConfig.Middlewares) + helperOptions := []limiterhelper.Option{ + limiterhelper.WithRequestItemsLimit(), + limiterhelper.WithMemorySizeLimit(), + } + httpMux := http.NewServeMux() if r.nextTraces != nil { - httpTracesReceiver := trace.New(r.nextTraces, r.obsrepHTTP) + httpTracesReceiver := trace.New(limiterhelper.WrapTraces(limiterProvider, r.nextTraces, helperOptions...), r.obsrepHTTP) httpMux.HandleFunc(r.cfg.HTTP.TracesURLPath, func(resp http.ResponseWriter, req *http.Request) { handleTraces(resp, req, httpTracesReceiver) }) } if r.nextMetrics != nil { - httpMetricsReceiver := metrics.New(r.nextMetrics, r.obsrepHTTP) + httpMetricsReceiver := metrics.New(limiterhelper.WrapMetrics(limiterProvider, r.nextMetrics, helperOptions...), r.obsrepHTTP) httpMux.HandleFunc(r.cfg.HTTP.MetricsURLPath, func(resp http.ResponseWriter, req *http.Request) { handleMetrics(resp, req, httpMetricsReceiver) }) } if r.nextLogs != nil { - httpLogsReceiver := logs.New(r.nextLogs, r.obsrepHTTP) + httpLogsReceiver := logs.New(limiterhelper.WrapLogs(limiterProvider, r.nextLogs, helperOptions...), r.obsrepHTTP) httpMux.HandleFunc(r.cfg.HTTP.LogsURLPath, func(resp http.ResponseWriter, req *http.Request) { handleLogs(resp, req, httpLogsReceiver) }) } if r.nextProfiles != nil { - httpProfilesReceiver := profiles.New(r.nextProfiles) + httpProfilesReceiver := profiles.New(limiterhelper.WrapProfiles(limiterProvider, r.nextProfiles, helperOptions...)) httpMux.HandleFunc(defaultProfilesURLPath, func(resp http.ResponseWriter, req *http.Request) { handleProfiles(resp, req, httpProfilesReceiver) })