Skip to content

Commit 57d9e38

Browse files
danielqsjdanielqsj
authored
Merge pull request #415 from errm/msk_iam_auth
Add support for authentication with AWS MSK via IAM
2 parents 477643d + 29aca29 commit 57d9e38

File tree

4 files changed

+66
-6
lines changed

4 files changed

+66
-6
lines changed

‎README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,8 @@ This image is configurable using different flags
115115
| sasl.handshake | true | Only set this to false if using a non-Kafka SASL proxy |
116116
| sasl.username | | SASL user name |
117117
| sasl.password | | SASL user password |
118-
| sasl.mechanism | | SASL mechanism can be plain, scram-sha512, scram-sha256 |
118+
| sasl.mechanism | plain | SASL SCRAM SHA algorithm: sha256 or sha512 or SASL mechanism: gssapi or awsiam |
119+
| sasl.aws-region | AWS_REGION env | The AWS region for IAM SASL authentication |
119120
| sasl.service-name | | Service name when using Kerberos Auth |
120121
| sasl.kerberos-config-path | | Kerberos config path |
121122
| sasl.realm | | Kerberos realm |

‎go.mod

+14-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.23
55
require (
66
github.com/IBM/sarama v1.43.3
77
github.com/alecthomas/kingpin/v2 v2.4.0
8+
github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0
89
github.com/krallistic/kazoo-go v0.0.0-20170526135507-a15279744f4e
910
github.com/pkg/errors v0.9.1
1011
github.com/prometheus/client_golang v1.20.0
@@ -16,6 +17,18 @@ require (
1617

1718
require (
1819
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
20+
github.com/aws/aws-sdk-go-v2 v1.19.0 // indirect
21+
github.com/aws/aws-sdk-go-v2/config v1.18.28 // indirect
22+
github.com/aws/aws-sdk-go-v2/credentials v1.13.27 // indirect
23+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 // indirect
24+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 // indirect
25+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 // indirect
26+
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 // indirect
27+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 // indirect
28+
github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 // indirect
29+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 // indirect
30+
github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 // indirect
31+
github.com/aws/smithy-go v1.13.5 // indirect
1932
github.com/beorn7/perks v1.0.1 // indirect
2033
github.com/cespare/xxhash/v2 v2.3.0 // indirect
2134
github.com/davecgh/go-spew v1.1.1 // indirect
@@ -44,7 +57,7 @@ require (
4457
github.com/xhit/go-str2duration/v2 v2.1.0 // indirect
4558
golang.org/x/crypto v0.26.0 // indirect
4659
golang.org/x/net v0.28.0 // indirect
47-
golang.org/x/sys v0.24.0 // indirect
60+
golang.org/x/sys v0.23.0 // indirect
4861
golang.org/x/text v0.17.0 // indirect
4962
google.golang.org/protobuf v1.34.2 // indirect
5063
)

‎go.sum

+32-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,32 @@ github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjH
44
github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE=
55
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 h1:t3eaIm0rUkzbrIewtiFmMK5RXHej2XnoXNhxVsAYUfg=
66
github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs=
7+
github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0 h1:UyjtGmO0Uwl/K+zpzPwLoXzMhcN9xmnR2nrqJoBrg3c=
8+
github.com/aws/aws-msk-iam-sasl-signer-go v1.0.0/go.mod h1:TJAXuFs2HcMib3sN5L0gUC+Q01Qvy3DemvA55WuC+iA=
9+
github.com/aws/aws-sdk-go-v2 v1.19.0 h1:klAT+y3pGFBU/qVf1uzwttpBbiuozJYWzNLHioyDJ+k=
10+
github.com/aws/aws-sdk-go-v2 v1.19.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
11+
github.com/aws/aws-sdk-go-v2/config v1.18.28 h1:TINEaKyh1Td64tqFvn09iYpKiWjmHYrG1fa91q2gnqw=
12+
github.com/aws/aws-sdk-go-v2/config v1.18.28/go.mod h1:nIL+4/8JdAuNHEjn/gPEXqtnS02Q3NXB/9Z7o5xE4+A=
13+
github.com/aws/aws-sdk-go-v2/credentials v1.13.27 h1:dz0yr/yR1jweAnsCx+BmjerUILVPQ6FS5AwF/OyG1kA=
14+
github.com/aws/aws-sdk-go-v2/credentials v1.13.27/go.mod h1:syOqAek45ZXZp29HlnRS/BNgMIW6uiRmeuQsz4Qh2UE=
15+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5 h1:kP3Me6Fy3vdi+9uHd7YLr6ewPxRL+PU6y15urfTaamU=
16+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.5/go.mod h1:Gj7tm95r+QsDoN2Fhuz/3npQvcZbkEf5mL70n3Xfluc=
17+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35 h1:hMUCiE3Zi5AHrRNGf5j985u0WyqI6r2NULhUfo0N/No=
18+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.35/go.mod h1:ipR5PvpSPqIqL5Mi82BxLnfMkHVbmco8kUwO2xrCi0M=
19+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29 h1:yOpYx+FTBdpk/g+sBU6Cb1H0U/TLEcYYp66mYqsPpcc=
20+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.29/go.mod h1:M/eUABlDbw2uVrdAn+UsI6M727qp2fxkp8K0ejcBDUY=
21+
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36 h1:8r5m1BoAWkn0TDC34lUculryf7nUF25EgIMdjvGCkgo=
22+
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.36/go.mod h1:Rmw2M1hMVTwiUhjwMoIBFWFJMhvJbct06sSidxInkhY=
23+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29 h1:IiDolu/eLmuB18DRZibj77n1hHQT7z12jnGO7Ze3pLc=
24+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.29/go.mod h1:fDbkK4o7fpPXWn8YAPmTieAMuB9mk/VgvW64uaUqxd4=
25+
github.com/aws/aws-sdk-go-v2/service/sso v1.12.13 h1:sWDv7cMITPcZ21QdreULwxOOAmE05JjEsT6fCDtDA9k=
26+
github.com/aws/aws-sdk-go-v2/service/sso v1.12.13/go.mod h1:DfX0sWuT46KpcqbMhJ9QWtxAIP1VozkDWf8VAkByjYY=
27+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13 h1:BFubHS/xN5bjl818QaroN6mQdjneYQ+AOx44KNXlyH4=
28+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.13/go.mod h1:BzqsVVFduubEmzrVtUFQQIQdFqvUItF8XUq2EnS8Wog=
29+
github.com/aws/aws-sdk-go-v2/service/sts v1.19.3 h1:e5mnydVdCVWxP+5rPAGi2PYxC7u2OZgH1ypC114H04U=
30+
github.com/aws/aws-sdk-go-v2/service/sts v1.19.3/go.mod h1:yVGZA1CPkmUhBdA039jXNJJG7/6t+G+EBWmFq23xqnY=
31+
github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
32+
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
733
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
834
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
935
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
@@ -27,6 +53,7 @@ github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
2753
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
2854
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
2955
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
56+
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
3057
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
3158
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
3259
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
@@ -51,6 +78,8 @@ github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh6
5178
github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs=
5279
github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY=
5380
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
81+
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
82+
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
5483
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
5584
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
5685
github.com/krallistic/kazoo-go v0.0.0-20170526135507-a15279744f4e h1:IWiVY66Xy9YrDZ28qJMt1UTlh6x9UGW0aDH/o58CSnA=
@@ -119,8 +148,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
119148
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
120149
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
121150
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
122-
golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg=
123-
golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
151+
golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM=
152+
golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
124153
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
125154
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
126155
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -138,6 +167,7 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h
138167
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
139168
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
140169
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
170+
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
141171
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
142172
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
143173
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

‎kafka_exporter.go

+18-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package main
22

33
import (
4+
"context"
45
"crypto/tls"
56
"crypto/x509"
67
"flag"
@@ -16,6 +17,7 @@ import (
1617

1718
"github.com/IBM/sarama"
1819
kingpin "github.com/alecthomas/kingpin/v2"
20+
"github.com/aws/aws-msk-iam-sasl-signer-go/signer"
1921
"github.com/krallistic/kazoo-go"
2022
"github.com/pkg/errors"
2123
"github.com/prometheus/client_golang/prometheus"
@@ -89,6 +91,7 @@ type kafkaOpts struct {
8991
saslPassword string
9092
saslMechanism string
9193
saslDisablePAFXFast bool
94+
saslAwsRegion string
9295
useTLS bool
9396
tlsServerName string
9497
tlsCAFile string
@@ -117,6 +120,15 @@ type kafkaOpts struct {
117120
verbosityLogLevel int
118121
}
119122

123+
type MSKAccessTokenProvider struct {
124+
region string
125+
}
126+
127+
func (m *MSKAccessTokenProvider) Token() (*sarama.AccessToken, error) {
128+
token, _, err := signer.GenerateAuthToken(context.TODO(), m.region)
129+
return &sarama.AccessToken{Token: token}, err
130+
}
131+
120132
// CanReadCertAndKey returns true if the certificate and key files already exists,
121133
// otherwise returns false. If lost one of cert and key, returns error.
122134
func CanReadCertAndKey(certPath, keyPath string) (bool, error) {
@@ -188,10 +200,13 @@ func NewExporter(opts kafkaOpts, topicFilter string, topicExclude string, groupF
188200
if opts.saslDisablePAFXFast {
189201
config.Net.SASL.GSSAPI.DisablePAFXFAST = true
190202
}
203+
case "awsiam":
204+
config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeOAuth)
205+
config.Net.SASL.TokenProvider = &MSKAccessTokenProvider{region: opts.saslAwsRegion}
191206
case "plain":
192207
default:
193208
return nil, fmt.Errorf(
194-
`invalid sasl mechanism "%s": can only be "scram-sha256", "scram-sha512", "gssapi" or "plain"`,
209+
`invalid sasl mechanism "%s": can only be "scram-sha256", "scram-sha512", "gssapi", "awsiam" or "plain"`,
195210
opts.saslMechanism,
196211
)
197212
}
@@ -749,7 +764,8 @@ func main() {
749764
toFlagBoolVar("sasl.handshake", "Only set this to false if using a non-Kafka SASL proxy, default is true.", true, "true", &opts.useSASLHandshake)
750765
toFlagStringVar("sasl.username", "SASL user name.", "", &opts.saslUsername)
751766
toFlagStringVar("sasl.password", "SASL user password.", "", &opts.saslPassword)
752-
toFlagStringVar("sasl.mechanism", "The SASL SCRAM SHA algorithm sha256 or sha512 or gssapi as mechanism", "", &opts.saslMechanism)
767+
toFlagStringVar("sasl.aws-region", "The AWS region for IAM SASL authentication", os.Getenv("AWS_REGION"), &opts.saslAwsRegion)
768+
toFlagStringVar("sasl.mechanism", "SASL SCRAM SHA algorithm: sha256 or sha512 or SASL mechanism: gssapi or awsiam", "", &opts.saslMechanism)
753769
toFlagStringVar("sasl.service-name", "Service name when using kerberos Auth", "", &opts.serviceName)
754770
toFlagStringVar("sasl.kerberos-config-path", "Kerberos config path", "", &opts.kerberosConfigPath)
755771
toFlagStringVar("sasl.realm", "Kerberos realm", "", &opts.realm)

0 commit comments

Comments
 (0)