From 0fc9d551d9f198c4776b8a3c10b4f6e3f24adb8f Mon Sep 17 00:00:00 2001 From: Trim21 Date: Thu, 10 Oct 2024 20:16:45 +0800 Subject: [PATCH] build: upgrade aws sdk to v2 (#643) --- canal/event.go | 6 ++--- canal/on_user.go | 55 +++++++++++++++++++++------------------ go.mod | 14 ++++++++-- go.sum | 33 ++++++++++++++++------- internal/pkg/driver/s3.go | 26 +++++++----------- 5 files changed, 79 insertions(+), 55 deletions(-) diff --git a/canal/event.go b/canal/event.go index f880b784e..fec87c3d2 100644 --- a/canal/event.go +++ b/canal/event.go @@ -19,7 +19,7 @@ import ( "encoding/json" "sync/atomic" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/redis/go-redis/v9" "github.com/trim21/errgo" "go.uber.org/zap" @@ -38,7 +38,7 @@ func newEventHandler( redis *redis.Client, stream Stream, search search.Client, - s3 *s3.S3, + s3 *s3.Client, ) *eventHandler { return &eventHandler{ redis: redis, @@ -58,7 +58,7 @@ type eventHandler struct { log *zap.Logger search search.Client stream Stream - s3 *s3.S3 // optional, check nil before use + s3 *s3.Client // optional, check nil before use redis *redis.Client } diff --git a/canal/on_user.go b/canal/on_user.go index da16f7b7a..27c5af9b1 100644 --- a/canal/on_user.go +++ b/canal/on_user.go @@ -20,8 +20,10 @@ import ( "encoding/json" "fmt" "strings" + "time" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/samber/lo" "github.com/trim21/errgo" "go.uber.org/zap" @@ -91,34 +93,37 @@ func (e *eventHandler) clearImageCache(avatar string) { e.log.Debug("clear image for prefix", zap.String("avatar", avatar), zap.String("prefix", p)) - err := e.s3.ListObjectsV2PagesWithContext(context.Background(), - &s3.ListObjectsV2Input{Bucket: &e.config.S3ImageResizeBucket, Prefix: &p}, - func(output *s3.ListObjectsV2Output, b bool) bool { - if len(output.Contents) == 0 { - return false - } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() - _, err := e.s3.DeleteObjects(&s3.DeleteObjectsInput{ - Bucket: &e.config.S3ImageResizeBucket, - Delete: &s3.Delete{ - Objects: lo.Map(output.Contents, func(item *s3.Object, index int) *s3.ObjectIdentifier { - return &s3.ObjectIdentifier{ - Key: item.Key, - } - }), - }, - }) + pages := s3.NewListObjectsV2Paginator( + e.s3, + &s3.ListObjectsV2Input{Bucket: &e.config.S3ImageResizeBucket, Prefix: &p}, + ) - if err != nil { - e.log.Error("failed to clear s3 cached image", zap.Error(err)) - } + for pages.HasMorePages() { + output, err := pages.NextPage(ctx) + if err != nil { + break + } - return true - }, - ) + if len(output.Contents) == 0 { + break + } - if err != nil { - e.log.Error("failed to clear s3 cached image", zap.Error(err)) + _, err = e.s3.DeleteObjects(ctx, &s3.DeleteObjectsInput{ + Bucket: &e.config.S3ImageResizeBucket, + Delete: &types.Delete{ + Objects: lo.Map(output.Contents, func(item types.Object, index int) types.ObjectIdentifier { + return types.ObjectIdentifier{ + Key: item.Key, + } + }), + }, + }) + if err != nil { + e.log.Error("failed to clear s3 cached image", zap.Error(err)) + } } } diff --git a/go.mod b/go.mod index c12045895..d30917dd9 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,9 @@ go 1.23.1 require ( github.com/avast/retry-go/v4 v4.6.0 - github.com/aws/aws-sdk-go v1.55.5 + github.com/aws/aws-sdk-go-v2 v1.32.2 + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 + github.com/aws/aws-sdk-go-v2/service/s3 v1.65.2 github.com/davecgh/go-spew v1.1.1 github.com/elliotchance/phpserialize v1.4.0 github.com/go-playground/locales v0.14.1 @@ -49,6 +51,15 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/BurntSushi/toml v1.4.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -60,7 +71,6 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.10 // indirect diff --git a/go.sum b/go.sum index 018e8d75f..b08d60829 100644 --- a/go.sum +++ b/go.sum @@ -26,9 +26,31 @@ github.com/avast/retry-go/v4 v4.6.0 h1:K9xNA+KeB8HHc2aWFuLb25Offp+0iVRXEvFx8IinR github.com/avast/retry-go/v4 v4.6.0/go.mod h1:gvWlPhBVsvBbLkVGDg/KwvBv0bEkCOLRRSHKIr2PyOE= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= -github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21 h1:7edmS3VOBDhK00b/MwGtGglCm7hhwNYnjJs/PgFdMQE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.21/go.mod h1:Q9o5h4HoIWG8XfzxqiuK/CGUbepCJ8uTlaE3bAbxytQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2 h1:4FMHqLfk0efmTqhXVRL5xYRqlEBNBiRI7N6w4jsEdd4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.2/go.mod h1:LWoqeWlK9OZeJxsROW2RqrSPvQHKTpp69r/iDjwsSaw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2 h1:t7iUP9+4wdc5lt3E41huP+GvQZJD38WLsgVp4iOtAjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.2/go.mod h1:/niFCtmuQNxqx9v8WAPq5qh7EH25U4BF6tjoyq9bObM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.2 h1:yi8m+jepdp6foK14xXLGkYBenxnlcfJ45ka4Pg7fDSQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.65.2/go.mod h1:cB6oAuus7YXRZhWCc1wIwPywwZ1XwweNp2TVAEGYeB8= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -203,10 +225,6 @@ github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -644,10 +662,7 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pkg/driver/s3.go b/internal/pkg/driver/s3.go index 97499fe68..84a943349 100644 --- a/internal/pkg/driver/s3.go +++ b/internal/pkg/driver/s3.go @@ -15,30 +15,24 @@ package driver import ( - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/samber/lo" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/bangumi/server/config" ) -func NewS3(c config.AppConfig) (*s3.S3, error) { +func NewS3(c config.AppConfig) (*s3.Client, error) { if c.S3EntryPoint == "" { return nil, nil //nolint:nilnil } - cred := credentials.NewStaticCredentials(c.S3AccessKey, c.S3SecretKey, "") - s := lo.Must(session.NewSession(&aws.Config{ - Credentials: cred, - Endpoint: &c.S3EntryPoint, - Region: lo.ToPtr("us-east-1"), - DisableSSL: lo.ToPtr(true), - S3ForcePathStyle: lo.ToPtr(true), - })) - - svc := s3.New(s) + svc := s3.New(s3.Options{ + BaseEndpoint: aws.String(c.S3EntryPoint), + Region: "us-east-1", + UsePathStyle: true, + Credentials: credentials.NewStaticCredentialsProvider(c.S3AccessKey, c.S3SecretKey, ""), + }) return svc, nil }