From d916bec1bca025ebfa67be5798d1cb0c77b4366a Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Fri, 12 Jan 2024 14:17:57 +0530 Subject: [PATCH 01/16] feat: added watermill --- .env.example | 4 + cli/api.go | 7 +- cli/main.go | 5 +- cli/worker.go | 106 +++++++++++++++++ config/main.go | 4 + config/mq.go | 14 +++ controllers/api/v1/user_controller.go | 9 +- go.mod | 43 ++++++- go.sum | 163 +++++++++++++++++++++++++- pkg/watermill/sender.go | 62 ++++++++++ pkg/watermill/worker.go | 121 +++++++++++++++++++ routes/main.go | 9 +- 12 files changed, 535 insertions(+), 12 deletions(-) create mode 100644 cli/worker.go create mode 100644 config/mq.go create mode 100644 pkg/watermill/sender.go create mode 100644 pkg/watermill/worker.go diff --git a/.env.example b/.env.example index 7a081ab..c7b3608 100644 --- a/.env.example +++ b/.env.example @@ -22,3 +22,7 @@ SQLITE_FILEPATH=database/golang-api.db #JWT JWT_SECRET=ThisIsKey + +# Message queue config +MQ_DIALECT=amqp +AMQB_URI=amqp://guest:guest@localhost:5672/ \ No newline at end of file diff --git a/cli/api.go b/cli/api.go index d3b74e5..c782f1b 100644 --- a/cli/api.go +++ b/cli/api.go @@ -10,6 +10,7 @@ import ( "github.com/Improwised/golang-api/config" "github.com/Improwised/golang-api/database" "github.com/Improwised/golang-api/pkg/events" + "github.com/Improwised/golang-api/pkg/watermill" "github.com/Improwised/golang-api/routes" "github.com/gofiber/fiber/v2" "github.com/spf13/cobra" @@ -43,8 +44,12 @@ func GetAPICommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { return err } + pub, err := watermill.InitSender(cfg) + if err != nil { + return err + } // setup routes - err = routes.Setup(app, db, logger, cfg, events, promMetrics) + err = routes.Setup(app, db, logger, cfg, events, promMetrics, pub) if err != nil { return err } diff --git a/cli/main.go b/cli/main.go index 01f6e90..22bd254 100644 --- a/cli/main.go +++ b/cli/main.go @@ -11,7 +11,10 @@ func Init(cfg config.AppConfig, logger *zap.Logger) error { migrationCmd := GetMigrationCommandDef(cfg) apiCmd := GetAPICommandDef(cfg, logger) + workerCmd:=GetWorkerCommandDef(cfg, logger) + workerCmd.PersistentFlags().String("topic", "demo", "Topic to subscribe") + rootCmd := &cobra.Command{Use: "golang-api"} - rootCmd.AddCommand(&migrationCmd, &apiCmd) + rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd) return rootCmd.Execute() } diff --git a/cli/worker.go b/cli/worker.go new file mode 100644 index 0000000..37c363c --- /dev/null +++ b/cli/worker.go @@ -0,0 +1,106 @@ +package cli + +import ( + "encoding/json" + + "go.uber.org/zap" + + "github.com/Improwised/golang-api/config" + "github.com/Improwised/golang-api/database" + "github.com/Improwised/golang-api/models" + "github.com/Improwised/golang-api/pkg/events" + "github.com/Improwised/golang-api/pkg/structs" + "github.com/Improwised/golang-api/pkg/watermill" + "github.com/Improwised/golang-api/services" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/spf13/cobra" +) + +// GetAPICommandDef runs app + +type worker struct { + userModel *services.UserService + events *events.Events +} + +func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { + + workerCommand := cobra.Command{ + Use: "worker", + Short: "To start worker", + Long: `To start worker`, + RunE: func(cmd *cobra.Command, args []string) error { + + events := events.NewEventBus(logger) + + goqu, err := database.Connect(cfg.DB) + if err != nil { + return err + } + err = events.SubscribeAll() + if err != nil { + return err + } + + userModel, err := models.InitUserModel(goqu) + if err != nil { + return err + } + userSvc := services.NewUserService(&userModel) + + // Init worker + sub, err := watermill.InitWorker(cfg) + if err != nil { + return err + } + + // get topic from flag + topic, err := cmd.Flags().GetString("topic") + if err != nil { + return err + } + + // get retry count from flag + retryCount, err := cmd.Flags().GetInt("retry-count") + if err != nil { + return err + } + + // get delay from flag + delay, err := cmd.Flags().GetInt("delay") + if err != nil { + return err + } + // Init router for add middleware,retry count,etc + router, err := sub.InitRouter(cfg, retryCount, delay) + if err != nil { + return err + } + + w := worker{ + userModel: userSvc, + events: events, + } + switch topic { + case "create_user": + err = router.Run(topic, w.Process) + return err + } + return nil + }, + } + + return workerCommand +} + +func (w *worker) Process(msg *message.Message) error { + var userReq structs.ReqRegisterUser + + err := json.Unmarshal(msg.Payload, &userReq) + if err != nil { + return err + } + _, err = w.userModel.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, w.events) + + return err +} diff --git a/config/main.go b/config/main.go index 7567d73..6376c59 100644 --- a/config/main.go +++ b/config/main.go @@ -19,7 +19,11 @@ type AppConfig struct { Env string `envconfig:"APP_ENV"` Port string `envconfig:"APP_PORT"` Secret string `envconfig:"JWT_SECRET"` + MQDialect string `envconfig:"MQ_DIALECT"` DB DBConfig + AMQB AmqpConfig + Redis RedisConfig + Kafka KafkaConfig } // GetConfig Collects all configs diff --git a/config/mq.go b/config/mq.go new file mode 100644 index 0000000..f5b81f6 --- /dev/null +++ b/config/mq.go @@ -0,0 +1,14 @@ +package config + +type RedisConfig struct { + RedisUrl string `envconfig:"REDIS_URI"` + ConsumerGroup string `envconfig:"CONSUMER_GROUP"` +} + +type AmqpConfig struct { + AmqbUrl string `envconfig:"AMQB_URI"` +} +type KafkaConfig struct { + KafkaBroker []string `envconfig:"KAFKA_BROKER"` + ConsumerGroup string `envconfig:"CONSUMER_GROUP"` +} diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index f5d950d..ceb1e13 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -9,6 +9,7 @@ import ( "github.com/Improwised/golang-api/models" "github.com/Improwised/golang-api/pkg/events" "github.com/Improwised/golang-api/pkg/structs" + "github.com/Improwised/golang-api/pkg/watermill" "github.com/Improwised/golang-api/services" "github.com/Improwised/golang-api/utils" "github.com/doug-martin/goqu/v9" @@ -23,10 +24,11 @@ type UserController struct { userService *services.UserService logger *zap.Logger event *events.Events + pub *watermill.WatermillPubliser } // NewUserController returns a user -func NewUserController(goqu *goqu.Database, logger *zap.Logger, event *events.Events) (*UserController, error) { +func NewUserController(goqu *goqu.Database, logger *zap.Logger, event *events.Events, pub *watermill.WatermillPubliser) (*UserController, error) { userModel, err := models.InitUserModel(goqu) if err != nil { return nil, err @@ -38,6 +40,7 @@ func NewUserController(goqu *goqu.Database, logger *zap.Logger, event *events.Ev userService: userSvc, logger: logger, event: event, + pub: pub, }, nil } @@ -97,6 +100,10 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { return utils.JSONFail(c, http.StatusBadRequest, utils.ValidatorErrorString(err)) } + err=ctrl.pub.PublishMessages("create_user", c.Body()) + if err != nil { + return utils.JSONFail(c, http.StatusBadRequest, err.Error()) + } user, err := ctrl.userService.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, ctrl.event) if err != nil { ctrl.logger.Error("error while insert user", zap.Error(err)) diff --git a/go.mod b/go.mod index 741a6cc..a6b6b76 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,17 @@ module github.com/Improwised/golang-api -go 1.19 +go 1.21 + +toolchain go1.21.5 require ( clevergo.tech/jsend v1.1.3 + github.com/Shopify/sarama v1.38.0 + github.com/ThreeDotsLabs/watermill v1.3.5 + github.com/ThreeDotsLabs/watermill-amqp v1.1.4 + github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.1 + github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0 + github.com/ThreeDotsLabs/watermill-redisstream v1.2.2 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef github.com/doug-martin/goqu/v9 v9.18.0 github.com/getsentry/sentry-go v0.25.0 @@ -17,6 +25,7 @@ require ( github.com/lib/pq v1.10.9 github.com/mattn/go-sqlite3 v1.14.17 github.com/prometheus/client_golang v1.16.0 + github.com/redis/go-redis/v9 v9.2.1 github.com/rs/xid v1.5.0 github.com/rubenv/sql-migrate v1.5.2 github.com/samber/lo v1.38.1 @@ -29,18 +38,35 @@ require ( ) require ( + github.com/Rican7/retry v0.3.1 // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v3 v3.2.2 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/eapache/go-resiliency v1.3.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect + github.com/eapache/queue v1.1.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/klauspost/compress v1.16.3 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/lestrrat-go/blackmagic v1.0.1 // indirect @@ -48,27 +74,40 @@ require ( github.com/lestrrat-go/httprc v1.0.4 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect + github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect + github.com/rabbitmq/amqp091-go v1.9.0 // indirect + github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/segmentio/asm v1.2.0 // indirect + github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/streadway/amqp v1.0.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.48.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect + github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.31.0 // indirect + go.opentelemetry.io/otel v1.6.1 // indirect + go.opentelemetry.io/otel/trace v1.6.1 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/text v0.11.0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index a904dcc..2d69cd5 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,45 @@ clevergo.tech/jsend v1.1.3 h1:noSA5WtIrEfX4gKxlJB/EQTpbHUxkK2E+nR9pguMZsI= clevergo.tech/jsend v1.1.3/go.mod h1:0w6SXsvj2f62Dy8fHBHFrMWQMB5K2uIzfiDFIMFh82k= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/Rican7/retry v0.3.1 h1:scY4IbO8swckzoA/11HgBwaZRJEyY9vaNJshcdhp1Mc= +github.com/Rican7/retry v0.3.1/go.mod h1:CxSDrhAyXmTMeEuRAnArMu1FHu48vtfjLREWqVl7Vw0= +github.com/Shopify/sarama v1.32.0/go.mod h1:+EmJJKZWVT/faR9RcOxJerP+LId4iWdQPBGLy1Y1Njs= +github.com/Shopify/sarama v1.38.0 h1:Q81EWxDT2Xs7kCaaiDGV30GyNCWd6K1Xmd4k2qpTWE8= +github.com/Shopify/sarama v1.38.0/go.mod h1:djdek3V4gS0N9LZ+OhfuuM6rE1bEKeDffYY8UvsRNyM= +github.com/Shopify/toxiproxy/v2 v2.3.0/go.mod h1:KvQTtB6RjCJY4zqNJn7C7JDFgsG5uoHYDirfUfpIm0c= +github.com/Shopify/toxiproxy/v2 v2.5.0 h1:i4LPT+qrSlKNtQf5QliVjdP08GyAH8+BUIc9gT0eahc= +github.com/Shopify/toxiproxy/v2 v2.5.0/go.mod h1:yhM2epWtAmel9CB8r2+L+PCmhH6yH2pITaPAo7jxJl0= +github.com/ThreeDotsLabs/watermill v1.3.5 h1:50JEPEhMGZQMh08ct0tfO1PsgMOAOhV3zxK2WofkbXg= +github.com/ThreeDotsLabs/watermill v1.3.5/go.mod h1:O/u/Ptyrk5MPTxSeWM5vzTtZcZfxXfO9PK9eXTYiFZY= +github.com/ThreeDotsLabs/watermill-amqp v1.1.4 h1:vOdc8a0m0sMPAJZ2CMLx5a+fwlgeeojOFPwgj7+nlJA= +github.com/ThreeDotsLabs/watermill-amqp v1.1.4/go.mod h1:5RtpKNTriXCWQZ67YDg1G7qsphZoUue/EWOmQqTZi3Q= +github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.1 h1:OxvMB2/3YtcQuC7quC+CGmFpGz9oaxP2ef5wkp+R2oM= +github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.1/go.mod h1:MCNoh0HUg4w0bY64on9BnhUodHeimz8+vMfXrzyuWN8= +github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0 h1:/KYEjLlLx6nW3jn6AEcwAlWkPWP62zi/sUsEP4uKkZE= +github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0/go.mod h1:w+9jhI7x5ZP67ceSUIIpkgLzjAakotfHX4sWyqsKVjs= +github.com/ThreeDotsLabs/watermill-redisstream v1.2.2 h1:/fFHagJiObMBbYIDrygRoAq+RxqLPcQZdGi6b0ViG08= +github.com/ThreeDotsLabs/watermill-redisstream v1.2.2/go.mod h1:ZRe0VpA0Ho/4MESUrXdqJMaWtiWhi4emxIYpqsxi98Y= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef/go.mod h1:JS7hed4L1fj0hXcyEejnW57/7LCetXggd+vwrRnYeII= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= +github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -19,13 +47,31 @@ github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPc github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/denisenkom/go-mssqldb v0.10.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/doug-martin/goqu/v9 v9.18.0 h1:/6bcuEtAe6nsSMVK/M+fOiXUNfyFF3yYtE07DBPFMYY= github.com/doug-martin/goqu/v9 v9.18.0/go.mod h1:nf0Wc2/hV3gYK9LiyqIrzBEVGlI8qW3GuDCEobC4wBQ= +github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= +github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= @@ -36,8 +82,11 @@ github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/gobuffalo/logger v1.0.6 h1:nnZNpxYo0zx+Aj9RfMPBm+x9zAU2OayFh/xrAWi34HU= +github.com/gobuffalo/logger v1.0.6/go.mod h1:J31TBEHR1QLV2683OXTAItYIg8pv2JMHnF/quuAbMjs= github.com/gobuffalo/packd v1.0.1 h1:U2wXfRr4E9DH8IdsDLlRFwTZTK7hLfq9qT/QHXGVe/0= +github.com/gobuffalo/packd v1.0.1/go.mod h1:PP2POP3p3RXGz7Jh6eYEf93S7vA2za6xM7QT85L4+VY= github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= +github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofiber/adaptor/v2 v2.2.1 h1:givE7iViQWlsTR4Jh7tB4iXzrlKBgiraB/yTdHs9Lv4= @@ -46,25 +95,65 @@ github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9c github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/gokrb5/v8 v8.4.3 h1:iTonLeSJOn7MVUtyMT+arAn5AKAPrkilzhGw8wE/Tq8= +github.com/jcmturner/gokrb5/v8 v8.4.3/go.mod h1:dqRwJGXznQrzw6cWmyo6kH+E7jksEQG/CyVWsJEsJO0= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= +github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.3 h1:XuJt9zzcnaz6a16/OU53ZjWp/v7/42WcR5t2a0PcNQY= github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80= @@ -83,9 +172,14 @@ github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmt github.com/lib/pq v1.10.1/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= +github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/markbates/errx v1.1.0 h1:QDFeR+UP95dO12JgW+tgi2UVfo0V8YBHiUIOaeBPiEI= +github.com/markbates/errx v1.1.0/go.mod h1:PLa46Oex9KNbVDZhKel8v1OT7hD5JZ2eI7AHhA0wswc= github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY= +github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI= github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -98,11 +192,19 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6 github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= +github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= +github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= @@ -111,29 +213,47 @@ github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rabbitmq/amqp091-go v1.9.0 h1:qrQtyzB4H8BQgEuJwhmVQqVHB9O4+MNDJCCAcpc3Aoo= +github.com/rabbitmq/amqp091-go v1.9.0/go.mod h1:+jPrT9iY2eLjRaMSRHUhc3z14E/l85kv/f+6luSD3pc= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= +github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg= +github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= github.com/rubenv/sql-migrate v1.5.2/go.mod h1:H38GW8Vqf8F0Su5XignRyaRcbXbJunSWxs+kmzlg0Is= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= +github.com/streadway/amqp v1.0.0/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -141,16 +261,29 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.48.0 h1:oJWvHb9BIZToTQS3MuQ2R3bJZiNSa2KiNdeI8A+79Tc= github.com/valyala/fasthttp v1.48.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= +github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= +github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.31.0 h1:J8jI81RCB7U9a3qsTZXM/38XrvbLJCye6J32bfQctYY= +go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.31.0/go.mod h1:72+cPzsW6geApbceSLMbZtYZeGMgtRDw5TcSEsdGlhc= +go.opentelemetry.io/otel v1.6.1 h1:6r1YrcTenBvYa1x491d0GGpTVBsNECmrc/K6b+zDeis= +go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ= +go.opentelemetry.io/otel/trace v1.6.1 h1:f8c93l5tboBYZna1nWk0W9DYyMzJXDWdZcJZ0Kb400U= +go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -158,7 +291,10 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= @@ -167,10 +303,15 @@ golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= @@ -178,11 +319,15 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -198,8 +343,11 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= @@ -212,17 +360,26 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/watermill/sender.go b/pkg/watermill/sender.go new file mode 100644 index 0000000..ab441b7 --- /dev/null +++ b/pkg/watermill/sender.go @@ -0,0 +1,62 @@ +package watermill + +import ( + "github.com/Improwised/golang-api/config" + "github.com/ThreeDotsLabs/watermill" + "github.com/ThreeDotsLabs/watermill-amqp/v2/pkg/amqp" + + "github.com/ThreeDotsLabs/watermill-kafka/v2/pkg/kafka" + "github.com/ThreeDotsLabs/watermill-redisstream/pkg/redisstream" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/redis/go-redis/v9" +) + +// GetAPICommandDef runs app +type WatermillPubliser struct { + publisher message.Publisher +} + +func InitSender(cfg config.AppConfig) (*WatermillPubliser, error) { + switch cfg.MQDialect { + case "amqp": + amqpConfig := amqp.NewDurableQueueConfig(cfg.AMQB.AmqbUrl) + publisher, err := amqp.NewPublisher(amqpConfig, watermill.NewStdLogger(false, false)) + return &WatermillPubliser{publisher: publisher}, err + + case "redis": + pubClient := redis.NewClient(&redis.Options{ + Addr: cfg.Redis.RedisUrl, + }) + publisher, err := redisstream.NewPublisher( + redisstream.PublisherConfig{ + Client: pubClient, + Marshaller: redisstream.DefaultMarshallerUnmarshaller{}, + }, + watermill.NewStdLogger(false, false), + ) + + return &WatermillPubliser{publisher: publisher}, err + + case "kafka": + publisher, err := kafka.NewPublisher( + kafka.PublisherConfig{ + Brokers: cfg.Kafka.KafkaBroker, + Marshaler: kafka.DefaultMarshaler{}, + OverwriteSaramaConfig: kafka.DefaultSaramaSyncPublisherConfig(), + }, + watermill.NewStdLogger(false, false), + ) + return &WatermillPubliser{publisher: publisher}, err + + default: + return &WatermillPubliser{}, nil + } + +} +func (wp *WatermillPubliser) PublishMessages(topic string, data []byte) error { + msg := message.NewMessage(watermill.NewUUID(), data) + if err := wp.publisher.Publish(topic, msg); err != nil { + return err + } + return nil +} diff --git a/pkg/watermill/worker.go b/pkg/watermill/worker.go new file mode 100644 index 0000000..49b6158 --- /dev/null +++ b/pkg/watermill/worker.go @@ -0,0 +1,121 @@ +package watermill + +import ( + "context" + "log" + "time" + + "github.com/Improwised/golang-api/config" + + "github.com/Shopify/sarama" + "github.com/ThreeDotsLabs/watermill" + "github.com/ThreeDotsLabs/watermill-amqp/pkg/amqp" + + "github.com/ThreeDotsLabs/watermill-kafka/v2/pkg/kafka" + "github.com/ThreeDotsLabs/watermill-redisstream/pkg/redisstream" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/ThreeDotsLabs/watermill/message/router/middleware" + "github.com/ThreeDotsLabs/watermill/message/router/plugin" + "github.com/redis/go-redis/v9" +) + +var logger = watermill.NewStdLogger(true, false) + +type WatermillSubscriber struct { + Subscriber message.Subscriber + Router *message.Router +} + +func InitWorker(cfg config.AppConfig) (*WatermillSubscriber, error) { + switch cfg.MQDialect { + case "amqp": + amqpConfig := amqp.NewDurableQueueConfig(cfg.AMQB.AmqbUrl) + subscriber, err := amqp.NewSubscriber( + amqpConfig, + watermill.NewStdLogger(false, false), + ) + return &WatermillSubscriber{Subscriber: subscriber}, err + + case "redis": + subClient := redis.NewClient(&redis.Options{ + Addr: cfg.Redis.RedisUrl, + }) + subscriber, err := redisstream.NewSubscriber( + redisstream.SubscriberConfig{ + Client: subClient, + Unmarshaller: redisstream.DefaultMarshallerUnmarshaller{}, + ConsumerGroup: cfg.Redis.ConsumerGroup, + }, + watermill.NewStdLogger(false, false), + ) + return &WatermillSubscriber{Subscriber: subscriber}, err + + case "kafka": + saramaSubscriberConfig := kafka.DefaultSaramaSubscriberConfig() + saramaSubscriberConfig.Consumer.Offsets.Initial = sarama.OffsetOldest + subscriber, err := kafka.NewSubscriber( + kafka.SubscriberConfig{ + Brokers: cfg.Kafka.KafkaBroker, + Unmarshaler: kafka.DefaultMarshaler{}, + OverwriteSaramaConfig: saramaSubscriberConfig, + ConsumerGroup: cfg.Kafka.ConsumerGroup, + }, + watermill.NewStdLogger(false, false), + ) + if err != nil { + return nil, err + } + return &WatermillSubscriber{Subscriber: subscriber}, err + default: + return nil, nil + } +} + +func (ws *WatermillSubscriber) InitRouter(cfg config.AppConfig, delayTime, MaxRetry int) (*WatermillSubscriber, error) { + router, err := message.NewRouter(message.RouterConfig{}, logger) + if err != nil { + return nil, err + } + + pub, err := InitSender(cfg) + if err != nil { + return nil, err + } + + poq, err := middleware.PoisonQueue(pub.publisher, "poison_queue") + if err != nil { + return nil, err + } + router.AddPlugin(plugin.SignalsHandler) + router.AddMiddleware( + middleware.CorrelationID, + poq, + middleware.Retry{ + MaxRetries: MaxRetry, + Logger: logger, + MaxInterval: time.Millisecond * time.Duration(delayTime), + InitialInterval: time.Millisecond * time.Duration(delayTime), + Multiplier: 1, + OnRetryHook: func(retryNum int, delay time.Duration) { + log.Println("retry count :=", delay) + }, + }.Middleware, + + middleware.Recoverer, + ) + ws.Router = router + + return ws, nil +} + +func (ws *WatermillSubscriber) Run(topic string, handlerFunc message.NoPublishHandlerFunc) error { + ws.Router.AddNoPublisherHandler( + "counter", + topic, + ws.Subscriber, + handlerFunc, + ) + + err := ws.Router.Run(context.Background()) + return err +} diff --git a/routes/main.go b/routes/main.go index 249e9e7..3d2152e 100644 --- a/routes/main.go +++ b/routes/main.go @@ -12,6 +12,7 @@ import ( "github.com/Improwised/golang-api/middlewares" "github.com/Improwised/golang-api/pkg/events" pMetrics "github.com/Improwised/golang-api/pkg/prometheus" + "github.com/Improwised/golang-api/pkg/watermill" "github.com/doug-martin/goqu/v9" "github.com/gofiber/fiber/v2" ) @@ -19,7 +20,7 @@ import ( var mu sync.Mutex // Setup func -func Setup(app *fiber.App, goqu *goqu.Database, logger *zap.Logger, config config.AppConfig, events *events.Events, pMetrics *pMetrics.PrometheusMetrics) error { +func Setup(app *fiber.App, goqu *goqu.Database, logger *zap.Logger, config config.AppConfig, events *events.Events, pMetrics *pMetrics.PrometheusMetrics, pub *watermill.WatermillPubliser) error { mu.Lock() app.Use(middlewares.LogHandler(logger, pMetrics)) @@ -40,7 +41,7 @@ func Setup(app *fiber.App, goqu *goqu.Database, logger *zap.Logger, config confi return err } - err = setupUserController(v1, goqu, logger, middlewares, events) + err = setupUserController(v1, goqu, logger, middlewares, events, pub) if err != nil { return err } @@ -68,8 +69,8 @@ func setupAuthController(v1 fiber.Router, goqu *goqu.Database, logger *zap.Logge return nil } -func setupUserController(v1 fiber.Router, goqu *goqu.Database, logger *zap.Logger, middlewares middlewares.Middleware, events *events.Events) error { - userController, err := controller.NewUserController(goqu, logger, events) +func setupUserController(v1 fiber.Router, goqu *goqu.Database, logger *zap.Logger, middlewares middlewares.Middleware, events *events.Events, pub *watermill.WatermillPubliser) error { + userController, err := controller.NewUserController(goqu, logger, events, pub) if err != nil { return err } From 23e1012c786cc36cdba9415aa2fe4f3a57afd55e Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Fri, 19 Jan 2024 12:23:48 +0530 Subject: [PATCH 02/16] test: add auto generate code for register --- cli/main.go | 3 +- cli/worker.go | 61 ++------------ cli/worker/gob_register.go | 11 +++ cli/worker/struct.go | 33 ++++++++ cli/worker/worker.go | 112 ++++++++++++++++++++++++++ controllers/api/v1/user_controller.go | 27 ++++++- go.mod | 10 ++- go.sum | 23 +++--- pkg/watermill/sender.go | 3 +- pkg/watermill/worker.go | 4 - 10 files changed, 211 insertions(+), 76 deletions(-) create mode 100644 cli/worker/gob_register.go create mode 100644 cli/worker/struct.go create mode 100644 cli/worker/worker.go diff --git a/cli/main.go b/cli/main.go index 22bd254..f55407b 100644 --- a/cli/main.go +++ b/cli/main.go @@ -13,7 +13,8 @@ func Init(cfg config.AppConfig, logger *zap.Logger) error { workerCmd:=GetWorkerCommandDef(cfg, logger) workerCmd.PersistentFlags().String("topic", "demo", "Topic to subscribe") - + workerCmd.PersistentFlags().Int("delay", 100, "time intertval to retry") + workerCmd.PersistentFlags().Int("retry-count", 3, "Number of retry") rootCmd := &cobra.Command{Use: "golang-api"} rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd) return rootCmd.Execute() diff --git a/cli/worker.go b/cli/worker.go index 37c363c..2ce21a7 100644 --- a/cli/worker.go +++ b/cli/worker.go @@ -1,28 +1,16 @@ package cli import ( - "encoding/json" - "go.uber.org/zap" "github.com/Improwised/golang-api/config" - "github.com/Improwised/golang-api/database" - "github.com/Improwised/golang-api/models" - "github.com/Improwised/golang-api/pkg/events" - "github.com/Improwised/golang-api/pkg/structs" "github.com/Improwised/golang-api/pkg/watermill" - "github.com/Improwised/golang-api/services" - "github.com/ThreeDotsLabs/watermill/message" + "github.com/Improwised/golang-api/cli/worker" "github.com/spf13/cobra" ) // GetAPICommandDef runs app -type worker struct { - userModel *services.UserService - events *events.Events -} - func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { workerCommand := cobra.Command{ @@ -31,25 +19,8 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command Long: `To start worker`, RunE: func(cmd *cobra.Command, args []string) error { - events := events.NewEventBus(logger) - - goqu, err := database.Connect(cfg.DB) - if err != nil { - return err - } - err = events.SubscribeAll() - if err != nil { - return err - } - - userModel, err := models.InitUserModel(goqu) - if err != nil { - return err - } - userSvc := services.NewUserService(&userModel) - // Init worker - sub, err := watermill.InitWorker(cfg) + subscriber, err := watermill.InitWorker(cfg) if err != nil { return err } @@ -71,36 +42,18 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command if err != nil { return err } + // Init router for add middleware,retry count,etc - router, err := sub.InitRouter(cfg, retryCount, delay) + router, err := subscriber.InitRouter(cfg, delay, retryCount) if err != nil { return err } - w := worker{ - userModel: userSvc, - events: events, - } - switch topic { - case "create_user": - err = router.Run(topic, w.Process) - return err - } - return nil + err = router.Run(topic, worker.Process) + return err + }, } return workerCommand } - -func (w *worker) Process(msg *message.Message) error { - var userReq structs.ReqRegisterUser - - err := json.Unmarshal(msg.Payload, &userReq) - if err != nil { - return err - } - _, err = w.userModel.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, w.events) - - return err -} diff --git a/cli/worker/gob_register.go b/cli/worker/gob_register.go new file mode 100644 index 0000000..2cdcca5 --- /dev/null +++ b/cli/worker/gob_register.go @@ -0,0 +1,11 @@ +package worker + +import ( + "encoding/gob" +) + +func GobRegister() { + gob.Register(UpdateUser{}) + gob.Register(DeleteUser{}) + gob.Register(AddUser{}) +} diff --git a/cli/worker/struct.go b/cli/worker/struct.go new file mode 100644 index 0000000..cbb89f1 --- /dev/null +++ b/cli/worker/struct.go @@ -0,0 +1,33 @@ +package worker + +type UpdateUser struct { + ID string `json:"id"` + FirstName string `json:"first_name" ` + LastName string `json:"last_name"` + Email string `json:"email"` + Password string `json:"-"` + Roles string `json:"roles"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} +type DeleteUser struct { + ID string `json:"id"` + FirstName string `json:"first_name" ` + LastName string `json:"last_name"` + Email string `json:"email"` + Password string `json:"-"` + Roles string `json:"roles"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} +type AddUser struct { + ID string `json:"id"` + FirstName string `json:"first_name" ` + LastName string `json:"last_name"` + Email string `json:"email"` + Password string `json:"-"` + Roles string `json:"roles"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + diff --git a/cli/worker/worker.go b/cli/worker/worker.go new file mode 100644 index 0000000..41050b0 --- /dev/null +++ b/cli/worker/worker.go @@ -0,0 +1,112 @@ +package worker + +import ( + "bytes" + "encoding/gob" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "text/template" + + "github.com/ThreeDotsLabs/watermill/message" +) + +type Handler interface { + Handle(data []byte) +} + +func (w DeleteUser) Handle(data []byte) { + fmt.Println("Delete User") + fmt.Println(w) +} +func (w UpdateUser) Handle(data []byte) { + fmt.Println("Update User") + fmt.Println(w) +} + +func (w AddUser) Handle(data []byte) { + fmt.Println("add user") + fmt.Println(w) +} + +func Process(msg *message.Message) error { + // GobRegister() + buf := bytes.NewBuffer(msg.Payload) + dec := gob.NewDecoder(buf) + + var result Handler + err := dec.Decode(&result) + if err != nil { + fmt.Println("Error in decoding") + return err + } + result.Handle(msg.Payload) + return nil +} + +func init() { + GenerateGobCode() +} + +// ------------------------------------------------------------------------------------- +// generate gob_register.go file + +const outputTemplate = `package {{.PackageName}} +import ( + "encoding/gob" + "fmt" +) +func GobRegister() { + {{range .StructNames}}gob.Register({{.}}{}) + fmt.Println("Gob Register") + {{end}} +} +` + +func GenerateGobCode() { + fset := token.NewFileSet() + _, filename, _, ok := runtime.Caller(0) + if !ok { + fmt.Println("cannot get current file") + } + dir := filepath.Dir(filename) + node, err := parser.ParseFile(fset, dir+"/struct.go", nil, parser.ParseComments) + if err != nil { + fmt.Println(err) + } + + structNames := []string{} + ast.Inspect(node, func(n ast.Node) bool { + typeSpec, ok := n.(*ast.TypeSpec) + if !ok { + return true + } + + _, ok = typeSpec.Type.(*ast.StructType) + if !ok { + return true + } + + structNames = append(structNames, typeSpec.Name.Name) + return false + }) + + outputFile, err := os.Create(dir + "/gob_register.go") + if err != nil { + fmt.Println(err) + + } + + defer outputFile.Close() + + t := template.Must(template.New("").Parse(outputTemplate)) + t.Execute(outputFile, map[string]interface{}{ + "PackageName": node.Name.Name, + "StructNames": structNames, + }) + +} diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index ceb1e13..7af4ab0 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -1,10 +1,14 @@ package v1 import ( + "bytes" "database/sql" + "encoding/gob" "encoding/json" + "log" "net/http" + "github.com/Improwised/golang-api/cli/worker" "github.com/Improwised/golang-api/constants" "github.com/Improwised/golang-api/models" "github.com/Improwised/golang-api/pkg/events" @@ -99,11 +103,26 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { if err != nil { return utils.JSONFail(c, http.StatusBadRequest, utils.ValidatorErrorString(err)) } + // ----------------------------------------------------------------------------------------- + // send message to worker + var network bytes.Buffer + enc := gob.NewEncoder(&network) + var message worker.Handler - err=ctrl.pub.PublishMessages("create_user", c.Body()) + myEvent := worker.DeleteUser{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles} + registerStructDynamically(myEvent) + + message = myEvent + err = enc.Encode(&message) + if err != nil { + log.Fatal("encode: niche", err) + } + + err = ctrl.pub.PublishMessages("worker.User", "user", network.Bytes()) if err != nil { return utils.JSONFail(c, http.StatusBadRequest, err.Error()) } + // ----------------------------------------------------------------------------------------- user, err := ctrl.userService.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, ctrl.event) if err != nil { ctrl.logger.Error("error while insert user", zap.Error(err)) @@ -112,3 +131,9 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { return utils.JSONSuccess(c, http.StatusCreated, user) } + +func registerStructDynamically(structValue interface{}) { + // t := reflect.TypeOf(structValue) + // gob.Register(t) + gob.Register(structValue) +} diff --git a/go.mod b/go.mod index a6b6b76..5bccdaf 100644 --- a/go.mod +++ b/go.mod @@ -33,10 +33,12 @@ require ( github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.18.0 gopkg.in/go-playground/validator.v9 v9.31.0 ) +require golang.org/x/sync v0.6.0 // indirect + require ( github.com/Rican7/retry v0.3.1 // indirect github.com/andybalholm/brotli v1.0.5 // indirect @@ -104,9 +106,9 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect - golang.org/x/net v0.10.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/text v0.11.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect diff --git a/go.sum b/go.sum index 2d69cd5..c56695b 100644 --- a/go.sum +++ b/go.sum @@ -296,8 +296,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -313,14 +313,15 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY5RMuhpJ3cNnI0XpyFJD1iQRSM= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -336,14 +337,14 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -352,8 +353,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/pkg/watermill/sender.go b/pkg/watermill/sender.go index ab441b7..6f172e5 100644 --- a/pkg/watermill/sender.go +++ b/pkg/watermill/sender.go @@ -53,8 +53,9 @@ func InitSender(cfg config.AppConfig) (*WatermillPubliser, error) { } } -func (wp *WatermillPubliser) PublishMessages(topic string, data []byte) error { +func (wp *WatermillPubliser) PublishMessages(key,topic string, data []byte) error { msg := message.NewMessage(watermill.NewUUID(), data) + msg.Metadata.Set("Key", key) if err := wp.publisher.Publish(topic, msg); err != nil { return err } diff --git a/pkg/watermill/worker.go b/pkg/watermill/worker.go index 49b6158..fe1d886 100644 --- a/pkg/watermill/worker.go +++ b/pkg/watermill/worker.go @@ -2,7 +2,6 @@ package watermill import ( "context" - "log" "time" "github.com/Improwised/golang-api/config" @@ -96,9 +95,6 @@ func (ws *WatermillSubscriber) InitRouter(cfg config.AppConfig, delayTime, MaxRe MaxInterval: time.Millisecond * time.Duration(delayTime), InitialInterval: time.Millisecond * time.Duration(delayTime), Multiplier: 1, - OnRetryHook: func(retryNum int, delay time.Duration) { - log.Println("retry count :=", delay) - }, }.Middleware, middleware.Recoverer, From ddcc13a9b9863d2868cd47d5d1e900c23c7cfb8f Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Fri, 19 Jan 2024 12:35:45 +0530 Subject: [PATCH 03/16] fix: rename file --- cli/worker/worker.go | 2 +- pkg/watermill/{sender.go => publisher.go} | 0 pkg/watermill/{worker.go => subscriber.go} | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename pkg/watermill/{sender.go => publisher.go} (100%) rename pkg/watermill/{worker.go => subscriber.go} (100%) diff --git a/cli/worker/worker.go b/cli/worker/worker.go index 41050b0..b7de382 100644 --- a/cli/worker/worker.go +++ b/cli/worker/worker.go @@ -34,7 +34,7 @@ func (w AddUser) Handle(data []byte) { } func Process(msg *message.Message) error { - // GobRegister() + GobRegister() buf := bytes.NewBuffer(msg.Payload) dec := gob.NewDecoder(buf) diff --git a/pkg/watermill/sender.go b/pkg/watermill/publisher.go similarity index 100% rename from pkg/watermill/sender.go rename to pkg/watermill/publisher.go diff --git a/pkg/watermill/worker.go b/pkg/watermill/subscriber.go similarity index 100% rename from pkg/watermill/worker.go rename to pkg/watermill/subscriber.go From 71feca29c1fb13fcd941240bc1077d27c3fbe938 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Wed, 24 Jan 2024 18:19:01 +0530 Subject: [PATCH 04/16] fix: restrucher file and code --- .env.example | 17 +++- cli/api.go | 2 +- cli/dead_letter_queue.go | 45 +++++++++++ cli/main.go | 11 +-- cli/worker.go | 23 +++--- cli/worker/gob_register.go | 11 --- cli/worker/struct.go | 33 -------- cli/worker/worker.go | 112 -------------------------- cli/workers/user.go | 32 ++++++++ cli/workers/worker_handler.go | 44 ++++++++++ config/main.go | 5 +- config/mq.go | 16 +++- controllers/api/v1/user_controller.go | 41 +++------- pkg/watermill/publisher.go | 96 +++++++++++++++------- pkg/watermill/subscriber.go | 110 ++++++++++++++++--------- 15 files changed, 315 insertions(+), 283 deletions(-) create mode 100644 cli/dead_letter_queue.go delete mode 100644 cli/worker/gob_register.go delete mode 100644 cli/worker/struct.go delete mode 100644 cli/worker/worker.go create mode 100644 cli/workers/user.go create mode 100644 cli/workers/worker_handler.go diff --git a/.env.example b/.env.example index c7b3608..af8f392 100644 --- a/.env.example +++ b/.env.example @@ -24,5 +24,20 @@ SQLITE_FILEPATH=database/golang-api.db JWT_SECRET=ThisIsKey # Message queue config +MQ_TRACK=true +MQ_DEBUG=true +DEAD_LETTER_QUEUE=dead_queue + +# Rabbitmq MQ_DIALECT=amqp -AMQB_URI=amqp://guest:guest@localhost:5672/ \ No newline at end of file +AMQB_URI=amqp://guest:guest@localhost:5672/ + +# Redis +# MQ_DIALECT=redis +# REDIS_HOST= +# CONSUMER_GROUP= + +# kafka +# MQ_DIALECT=kafka +# KAFKA_BROKER= +# CONSUMER_GROUP= diff --git a/cli/api.go b/cli/api.go index c782f1b..2216b3c 100644 --- a/cli/api.go +++ b/cli/api.go @@ -44,7 +44,7 @@ func GetAPICommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { return err } - pub, err := watermill.InitSender(cfg) + pub, err := watermill.InitPubliser(cfg) if err != nil { return err } diff --git a/cli/dead_letter_queue.go b/cli/dead_letter_queue.go new file mode 100644 index 0000000..b238ebd --- /dev/null +++ b/cli/dead_letter_queue.go @@ -0,0 +1,45 @@ +package cli + +import ( + "fmt" + + "go.uber.org/zap" + + "github.com/Improwised/golang-api/config" + "github.com/Improwised/golang-api/pkg/watermill" + "github.com/ThreeDotsLabs/watermill/message" + "github.com/spf13/cobra" +) + +// GetAPICommandDef runs app +// ? need rename in file name or command name +func GetDeadQueueCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { + + workerCommand := cobra.Command{ + Use: "dead-letter-queue", + Short: "To start dead-letter queue", + Long: `This queue is used to store failed job from all worker`, + RunE: func(cmd *cobra.Command, args []string) error { + + // Init worker + subscriber, err := watermill.InitSubscriber(cfg) + if err != nil { + return err + } + + // run worker with topic(queue name) and process function + if err := subscriber.Run(cfg.MQ.DeadQueue, HandleFailJob); err != nil { + return err + } + + return nil + }, + } + return workerCommand +} + +func HandleFailJob(msg *message.Message) error { + fmt.Println("failed job", string(msg.Payload)) + // process here + return nil +} diff --git a/cli/main.go b/cli/main.go index f55407b..f871a25 100644 --- a/cli/main.go +++ b/cli/main.go @@ -10,12 +10,13 @@ import ( func Init(cfg config.AppConfig, logger *zap.Logger) error { migrationCmd := GetMigrationCommandDef(cfg) apiCmd := GetAPICommandDef(cfg, logger) - workerCmd:=GetWorkerCommandDef(cfg, logger) - workerCmd.PersistentFlags().String("topic", "demo", "Topic to subscribe") - workerCmd.PersistentFlags().Int("delay", 100, "time intertval to retry") - workerCmd.PersistentFlags().Int("retry-count", 3, "Number of retry") + workerCmd.PersistentFlags().Int("retry-delay", 100, "time intertval for two retry in ms") + workerCmd.PersistentFlags().Int("retry-count", 3, "number of retry") + workerCmd.PersistentFlags().String("topic", "", "topic name(queue name)") + + d:=GetDeadQueueCommandDef(cfg, logger) rootCmd := &cobra.Command{Use: "golang-api"} - rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd) + rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd,&d) return rootCmd.Execute() } diff --git a/cli/worker.go b/cli/worker.go index 2ce21a7..55196b0 100644 --- a/cli/worker.go +++ b/cli/worker.go @@ -3,9 +3,9 @@ package cli import ( "go.uber.org/zap" + "github.com/Improwised/golang-api/cli/workers" "github.com/Improwised/golang-api/config" "github.com/Improwised/golang-api/pkg/watermill" - "github.com/Improwised/golang-api/cli/worker" "github.com/spf13/cobra" ) @@ -18,42 +18,39 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command Short: "To start worker", Long: `To start worker`, RunE: func(cmd *cobra.Command, args []string) error { - - // Init worker - subscriber, err := watermill.InitWorker(cfg) + // Get details name from flag + topic, err := cmd.Flags().GetString("topic") if err != nil { return err } - // get topic from flag - topic, err := cmd.Flags().GetString("topic") + retryCount, err := cmd.Flags().GetInt("retry-count") if err != nil { return err } - // get retry count from flag - retryCount, err := cmd.Flags().GetInt("retry-count") + delay, err := cmd.Flags().GetInt("retry-delay") if err != nil { return err } - // get delay from flag - delay, err := cmd.Flags().GetInt("delay") + // Init worker + subscriber, err := watermill.InitSubscriber(cfg) if err != nil { return err } - // Init router for add middleware,retry count,etc + // init router for add middleware,retry count,etc router, err := subscriber.InitRouter(cfg, delay, retryCount) if err != nil { return err } - err = router.Run(topic, worker.Process) + // run worker with topic(queue name) and process function + err = router.Run(topic, workers.Process) return err }, } - return workerCommand } diff --git a/cli/worker/gob_register.go b/cli/worker/gob_register.go deleted file mode 100644 index 2cdcca5..0000000 --- a/cli/worker/gob_register.go +++ /dev/null @@ -1,11 +0,0 @@ -package worker - -import ( - "encoding/gob" -) - -func GobRegister() { - gob.Register(UpdateUser{}) - gob.Register(DeleteUser{}) - gob.Register(AddUser{}) -} diff --git a/cli/worker/struct.go b/cli/worker/struct.go deleted file mode 100644 index cbb89f1..0000000 --- a/cli/worker/struct.go +++ /dev/null @@ -1,33 +0,0 @@ -package worker - -type UpdateUser struct { - ID string `json:"id"` - FirstName string `json:"first_name" ` - LastName string `json:"last_name"` - Email string `json:"email"` - Password string `json:"-"` - Roles string `json:"roles"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` -} -type DeleteUser struct { - ID string `json:"id"` - FirstName string `json:"first_name" ` - LastName string `json:"last_name"` - Email string `json:"email"` - Password string `json:"-"` - Roles string `json:"roles"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` -} -type AddUser struct { - ID string `json:"id"` - FirstName string `json:"first_name" ` - LastName string `json:"last_name"` - Email string `json:"email"` - Password string `json:"-"` - Roles string `json:"roles"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` -} - diff --git a/cli/worker/worker.go b/cli/worker/worker.go deleted file mode 100644 index b7de382..0000000 --- a/cli/worker/worker.go +++ /dev/null @@ -1,112 +0,0 @@ -package worker - -import ( - "bytes" - "encoding/gob" - "fmt" - "go/ast" - "go/parser" - "go/token" - "os" - "path/filepath" - "runtime" - "text/template" - - "github.com/ThreeDotsLabs/watermill/message" -) - -type Handler interface { - Handle(data []byte) -} - -func (w DeleteUser) Handle(data []byte) { - fmt.Println("Delete User") - fmt.Println(w) -} -func (w UpdateUser) Handle(data []byte) { - fmt.Println("Update User") - fmt.Println(w) -} - -func (w AddUser) Handle(data []byte) { - fmt.Println("add user") - fmt.Println(w) -} - -func Process(msg *message.Message) error { - GobRegister() - buf := bytes.NewBuffer(msg.Payload) - dec := gob.NewDecoder(buf) - - var result Handler - err := dec.Decode(&result) - if err != nil { - fmt.Println("Error in decoding") - return err - } - result.Handle(msg.Payload) - return nil -} - -func init() { - GenerateGobCode() -} - -// ------------------------------------------------------------------------------------- -// generate gob_register.go file - -const outputTemplate = `package {{.PackageName}} -import ( - "encoding/gob" - "fmt" -) -func GobRegister() { - {{range .StructNames}}gob.Register({{.}}{}) - fmt.Println("Gob Register") - {{end}} -} -` - -func GenerateGobCode() { - fset := token.NewFileSet() - _, filename, _, ok := runtime.Caller(0) - if !ok { - fmt.Println("cannot get current file") - } - dir := filepath.Dir(filename) - node, err := parser.ParseFile(fset, dir+"/struct.go", nil, parser.ParseComments) - if err != nil { - fmt.Println(err) - } - - structNames := []string{} - ast.Inspect(node, func(n ast.Node) bool { - typeSpec, ok := n.(*ast.TypeSpec) - if !ok { - return true - } - - _, ok = typeSpec.Type.(*ast.StructType) - if !ok { - return true - } - - structNames = append(structNames, typeSpec.Name.Name) - return false - }) - - outputFile, err := os.Create(dir + "/gob_register.go") - if err != nil { - fmt.Println(err) - - } - - defer outputFile.Close() - - t := template.Must(template.New("").Parse(outputTemplate)) - t.Execute(outputFile, map[string]interface{}{ - "PackageName": node.Name.Name, - "StructNames": structNames, - }) - -} diff --git a/cli/workers/user.go b/cli/workers/user.go new file mode 100644 index 0000000..8006d3b --- /dev/null +++ b/cli/workers/user.go @@ -0,0 +1,32 @@ +package workers + +import ( + "log" + + helpers "github.com/Improwised/golang-api/helpers/smtp" +) + +type WelcomeMail struct { + FirstName string `json:"first_name" ` + LastName string `json:"last_name"` + Email string `json:"email"` + Roles string `json:"roles"` +} + +func (w WelcomeMail) Handle() error { + log.Println("sending mail") + + smtp := helpers.NewSMTPHelper("sandbox.smtp.mailtrap.io", "2525", "7628fa366c0257c", "2afb7200812272") + smtp.SetSubject("welocme") + smtp.SetPlainBody([]byte("welcome to our org")) + + smtp.SetSender("chintansakhiya00001@gmail.com") + smtp.SetReceivers([]string{"chintansakhiya00001@gmail.com"}) + + if err := smtp.SendMail(); err != nil { + return err + } + + log.Printf("mail send to %v", w.Email) + return nil +} diff --git a/cli/workers/worker_handler.go b/cli/workers/worker_handler.go new file mode 100644 index 0000000..67ae2a5 --- /dev/null +++ b/cli/workers/worker_handler.go @@ -0,0 +1,44 @@ +package workers + +import ( + "bytes" + "encoding/gob" + + "github.com/ThreeDotsLabs/watermill/message" +) + +func init() { + for _, v := range ListStruct() { + gob.Register(v) + } +} + +// Register all worker struct here befour run worker for proper unmarshalling +func ListStruct() []interface{} { + return []interface{}{ + WelcomeMail{}, + // ... + } +} + +// Handler interface for all worker struct +type Handler interface { + Handle() error +} + +// process all worker struct and call Handle function according to struct +func Process(msg *message.Message) error { + buf := bytes.NewBuffer(msg.Payload) + dec := gob.NewDecoder(buf) + + var result Handler + err := dec.Decode(&result) + if err != nil { + return err + } + if err := result.Handle(); err != nil { + return err + } + msg.Ack() + return nil +} diff --git a/config/main.go b/config/main.go index 6376c59..80701aa 100644 --- a/config/main.go +++ b/config/main.go @@ -19,11 +19,8 @@ type AppConfig struct { Env string `envconfig:"APP_ENV"` Port string `envconfig:"APP_PORT"` Secret string `envconfig:"JWT_SECRET"` - MQDialect string `envconfig:"MQ_DIALECT"` DB DBConfig - AMQB AmqpConfig - Redis RedisConfig - Kafka KafkaConfig + MQ MQConfig } // GetConfig Collects all configs diff --git a/config/mq.go b/config/mq.go index f5b81f6..fb852f7 100644 --- a/config/mq.go +++ b/config/mq.go @@ -1,7 +1,17 @@ package config +// TODO: add env to .env.example +type MQConfig struct { + Dialect string `envconfig:"MQ_DIALECT"` + Debug bool `envconfig:"MQ_DEBUG"` + Track bool `envconfig:"MQ_TRACK"` + DeadQueue string `envconfig:"DEAD_LETTER_QUEUE"` + Redis RedisConfig + Amqp AmqpConfig + Kafka KafkaConfig +} type RedisConfig struct { - RedisUrl string `envconfig:"REDIS_URI"` + RedisUrl string `envconfig:"REDIS_URI"` ConsumerGroup string `envconfig:"CONSUMER_GROUP"` } @@ -9,6 +19,6 @@ type AmqpConfig struct { AmqbUrl string `envconfig:"AMQB_URI"` } type KafkaConfig struct { - KafkaBroker []string `envconfig:"KAFKA_BROKER"` - ConsumerGroup string `envconfig:"CONSUMER_GROUP"` + KafkaBroker []string `envconfig:"KAFKA_BROKER"` + ConsumerGroup string `envconfig:"CONSUMER_GROUP"` } diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index 7af4ab0..512cdd4 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -1,14 +1,11 @@ package v1 import ( - "bytes" "database/sql" - "encoding/gob" "encoding/json" - "log" "net/http" - "github.com/Improwised/golang-api/cli/worker" + "github.com/Improwised/golang-api/cli/workers" "github.com/Improwised/golang-api/constants" "github.com/Improwised/golang-api/models" "github.com/Improwised/golang-api/pkg/events" @@ -103,37 +100,19 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { if err != nil { return utils.JSONFail(c, http.StatusBadRequest, utils.ValidatorErrorString(err)) } - // ----------------------------------------------------------------------------------------- - // send message to worker - var network bytes.Buffer - enc := gob.NewEncoder(&network) - var message worker.Handler - - myEvent := worker.DeleteUser{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles} - registerStructDynamically(myEvent) - - message = myEvent - err = enc.Encode(&message) - if err != nil { - log.Fatal("encode: niche", err) - } - - err = ctrl.pub.PublishMessages("worker.User", "user", network.Bytes()) - if err != nil { - return utils.JSONFail(c, http.StatusBadRequest, err.Error()) - } - // ----------------------------------------------------------------------------------------- + user, err := ctrl.userService.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, ctrl.event) if err != nil { ctrl.logger.Error("error while insert user", zap.Error(err)) return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) } - + + // publish job to queue + myEvent := workers.WelcomeMail{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Roles: userReq.Roles} + err = ctrl.pub.Publish("user", myEvent) + if err != nil { + return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) + } + return utils.JSONSuccess(c, http.StatusCreated, user) } - -func registerStructDynamically(structValue interface{}) { - // t := reflect.TypeOf(structValue) - // gob.Register(t) - gob.Register(structValue) -} diff --git a/pkg/watermill/publisher.go b/pkg/watermill/publisher.go index 6f172e5..de29ea0 100644 --- a/pkg/watermill/publisher.go +++ b/pkg/watermill/publisher.go @@ -1,6 +1,12 @@ package watermill import ( + "bytes" + "encoding/gob" + "fmt" + "log" + + "github.com/Improwised/golang-api/cli/workers" "github.com/Improwised/golang-api/config" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amqp/v2/pkg/amqp" @@ -11,53 +17,83 @@ import ( "github.com/redis/go-redis/v9" ) -// GetAPICommandDef runs app type WatermillPubliser struct { publisher message.Publisher } -func InitSender(cfg config.AppConfig) (*WatermillPubliser, error) { - switch cfg.MQDialect { +func InitPubliser(cfg config.AppConfig) (*WatermillPubliser, error) { + logger = watermill.NewStdLogger(cfg.MQ.Debug, cfg.MQ.Track) + switch cfg.MQ.Dialect { case "amqp": - amqpConfig := amqp.NewDurableQueueConfig(cfg.AMQB.AmqbUrl) - publisher, err := amqp.NewPublisher(amqpConfig, watermill.NewStdLogger(false, false)) - return &WatermillPubliser{publisher: publisher}, err + return initAmqpPub(cfg) case "redis": - pubClient := redis.NewClient(&redis.Options{ - Addr: cfg.Redis.RedisUrl, - }) - publisher, err := redisstream.NewPublisher( - redisstream.PublisherConfig{ - Client: pubClient, - Marshaller: redisstream.DefaultMarshallerUnmarshaller{}, - }, - watermill.NewStdLogger(false, false), - ) - - return &WatermillPubliser{publisher: publisher}, err + return initRedisPub(cfg) case "kafka": - publisher, err := kafka.NewPublisher( - kafka.PublisherConfig{ - Brokers: cfg.Kafka.KafkaBroker, - Marshaler: kafka.DefaultMarshaler{}, - OverwriteSaramaConfig: kafka.DefaultSaramaSyncPublisherConfig(), - }, - watermill.NewStdLogger(false, false), - ) - return &WatermillPubliser{publisher: publisher}, err + return initKafkaPub(cfg) default: return &WatermillPubliser{}, nil } } -func (wp *WatermillPubliser) PublishMessages(key,topic string, data []byte) error { - msg := message.NewMessage(watermill.NewUUID(), data) - msg.Metadata.Set("Key", key) + +// send message to queue using topic name +// +// struct must from worker package(/cli/workers) +func (wp *WatermillPubliser) Publish(topic string, data interface{}) error { + var network bytes.Buffer + enc := gob.NewEncoder(&network) + var handle workers.Handler + + handle, ok := data.(workers.Handler) + if !ok { + return fmt.Errorf("data is not of type workers.Handler") + } + + err := enc.Encode(&handle) + if err != nil { + log.Fatal("encode: niche", err) + } + msg := message.NewMessage(watermill.NewUUID(), network.Bytes()) + if err := wp.publisher.Publish(topic, msg); err != nil { return err } return nil } + +func initAmqpPub(cfg config.AppConfig) (*WatermillPubliser, error) { + amqpConfig := amqp.NewDurableQueueConfig(cfg.MQ.Amqp.AmqbUrl) + publisher, err := amqp.NewPublisher(amqpConfig, logger) + return &WatermillPubliser{publisher: publisher}, err +} + +// TODO: username/pass +func initRedisPub(cfg config.AppConfig) (*WatermillPubliser, error) { + pubClient := redis.NewClient(&redis.Options{ + Addr: cfg.MQ.Redis.RedisUrl, + }) + publisher, err := redisstream.NewPublisher( + redisstream.PublisherConfig{ + Client: pubClient, + Marshaller: redisstream.DefaultMarshallerUnmarshaller{}, + }, + logger, + ) + return &WatermillPubliser{publisher: publisher}, err +} + +func initKafkaPub(cfg config.AppConfig) (*WatermillPubliser, error) { + publisher, err := kafka.NewPublisher( + kafka.PublisherConfig{ + Brokers: cfg.MQ.Kafka.KafkaBroker, + Marshaler: kafka.DefaultMarshaler{}, + OverwriteSaramaConfig: kafka.DefaultSaramaSyncPublisherConfig(), + }, + logger, + ) + return &WatermillPubliser{publisher: publisher}, err +} + diff --git a/pkg/watermill/subscriber.go b/pkg/watermill/subscriber.go index fe1d886..34f114e 100644 --- a/pkg/watermill/subscriber.go +++ b/pkg/watermill/subscriber.go @@ -2,6 +2,7 @@ package watermill import ( "context" + "fmt" "time" "github.com/Improwised/golang-api/config" @@ -18,70 +19,45 @@ import ( "github.com/redis/go-redis/v9" ) -var logger = watermill.NewStdLogger(true, false) +var logger watermill.LoggerAdapter type WatermillSubscriber struct { Subscriber message.Subscriber Router *message.Router } -func InitWorker(cfg config.AppConfig) (*WatermillSubscriber, error) { - switch cfg.MQDialect { +// TODO: for redis and kafka username/password +// TODO: add other dialect +func InitSubscriber(cfg config.AppConfig) (*WatermillSubscriber, error) { + logger = watermill.NewStdLogger(cfg.MQ.Debug, cfg.MQ.Track) + switch cfg.MQ.Dialect { case "amqp": - amqpConfig := amqp.NewDurableQueueConfig(cfg.AMQB.AmqbUrl) - subscriber, err := amqp.NewSubscriber( - amqpConfig, - watermill.NewStdLogger(false, false), - ) - return &WatermillSubscriber{Subscriber: subscriber}, err + return initAmqpSub(cfg) case "redis": - subClient := redis.NewClient(&redis.Options{ - Addr: cfg.Redis.RedisUrl, - }) - subscriber, err := redisstream.NewSubscriber( - redisstream.SubscriberConfig{ - Client: subClient, - Unmarshaller: redisstream.DefaultMarshallerUnmarshaller{}, - ConsumerGroup: cfg.Redis.ConsumerGroup, - }, - watermill.NewStdLogger(false, false), - ) - return &WatermillSubscriber{Subscriber: subscriber}, err + return initRedisSub(cfg) case "kafka": - saramaSubscriberConfig := kafka.DefaultSaramaSubscriberConfig() - saramaSubscriberConfig.Consumer.Offsets.Initial = sarama.OffsetOldest - subscriber, err := kafka.NewSubscriber( - kafka.SubscriberConfig{ - Brokers: cfg.Kafka.KafkaBroker, - Unmarshaler: kafka.DefaultMarshaler{}, - OverwriteSaramaConfig: saramaSubscriberConfig, - ConsumerGroup: cfg.Kafka.ConsumerGroup, - }, - watermill.NewStdLogger(false, false), - ) - if err != nil { - return nil, err - } - return &WatermillSubscriber{Subscriber: subscriber}, err + return initKafkaSub(cfg) + default: return nil, nil } } +// InitRouter init router for add middleware,retry count,delay etc func (ws *WatermillSubscriber) InitRouter(cfg config.AppConfig, delayTime, MaxRetry int) (*WatermillSubscriber, error) { router, err := message.NewRouter(message.RouterConfig{}, logger) if err != nil { return nil, err } - pub, err := InitSender(cfg) + pub, err := InitPubliser(cfg) if err != nil { return nil, err } - poq, err := middleware.PoisonQueue(pub.publisher, "poison_queue") + poq, err := middleware.PoisonQueue(pub.publisher, cfg.MQ.DeadQueue) if err != nil { return nil, err } @@ -105,8 +81,20 @@ func (ws *WatermillSubscriber) InitRouter(cfg config.AppConfig, delayTime, MaxRe } func (ws *WatermillSubscriber) Run(topic string, handlerFunc message.NoPublishHandlerFunc) error { + if ws.Subscriber == nil { + return fmt.Errorf("subscriber is nil") + } + + if ws.Router == nil { + router, err := message.NewRouter(message.RouterConfig{}, logger) + if err != nil { + return err + } + ws.Router = router + } + ws.Router.AddNoPublisherHandler( - "counter", + "handler", topic, ws.Subscriber, handlerFunc, @@ -115,3 +103,47 @@ func (ws *WatermillSubscriber) Run(topic string, handlerFunc message.NoPublishHa err := ws.Router.Run(context.Background()) return err } + +func initAmqpSub(cfg config.AppConfig) (*WatermillSubscriber, error) { + amqpConfig := amqp.NewDurableQueueConfig(cfg.MQ.Amqp.AmqbUrl) + subscriber, err := amqp.NewSubscriber( + amqpConfig, + logger, + ) + return &WatermillSubscriber{Subscriber: subscriber}, err +} + +func initKafkaSub(cfg config.AppConfig) (*WatermillSubscriber, error) { + saramaSubscriberConfig := kafka.DefaultSaramaSubscriberConfig() + saramaSubscriberConfig.Consumer.Offsets.Initial = sarama.OffsetOldest + subscriber, err := kafka.NewSubscriber( + kafka.SubscriberConfig{ + Brokers: cfg.MQ.Kafka.KafkaBroker, + Unmarshaler: kafka.DefaultMarshaler{}, + OverwriteSaramaConfig: saramaSubscriberConfig, + ConsumerGroup: cfg.MQ.Kafka.ConsumerGroup, + }, + logger, + ) + if err != nil { + return nil, err + } + return &WatermillSubscriber{Subscriber: subscriber}, err +} + +func initRedisSub(cfg config.AppConfig) (*WatermillSubscriber, error) { + subClient := redis.NewClient(&redis.Options{ + Addr: cfg.MQ.Redis.RedisUrl, + // Username: , + // password: , + }) + subscriber, err := redisstream.NewSubscriber( + redisstream.SubscriberConfig{ + Client: subClient, + Unmarshaller: redisstream.DefaultMarshallerUnmarshaller{}, + ConsumerGroup: cfg.MQ.Redis.ConsumerGroup, + }, + logger, + ) + return &WatermillSubscriber{Subscriber: subscriber}, err +} From d29477748f9e73d26e1a0ba31ca5c20490374d45 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Thu, 25 Jan 2024 16:08:40 +0530 Subject: [PATCH 05/16] doc: update readme.md file --- .env.example | 10 ++-- README.md | 78 +++++++++++++++++++++++++-- cli/dead_letter_queue.go | 10 ++-- cli/main.go | 4 +- cli/worker.go | 2 +- cli/workers/worker_handler.go | 4 +- config/mq.go | 1 - controllers/api/v1/user_controller.go | 10 ++-- pkg/watermill/publisher.go | 23 ++++---- 9 files changed, 105 insertions(+), 37 deletions(-) diff --git a/.env.example b/.env.example index af8f392..ca8853a 100644 --- a/.env.example +++ b/.env.example @@ -24,13 +24,13 @@ SQLITE_FILEPATH=database/golang-api.db JWT_SECRET=ThisIsKey # Message queue config -MQ_TRACK=true -MQ_DEBUG=true -DEAD_LETTER_QUEUE=dead_queue +# MQ_TRACK=true +# MQ_DEBUG=true +# DEAD_LETTER_QUEUE=dead_queue # Rabbitmq -MQ_DIALECT=amqp -AMQB_URI=amqp://guest:guest@localhost:5672/ +# MQ_DIALECT=amqp +# AMQB_URI=amqp://guest:guest@localhost:5672/ # Redis # MQ_DIALECT=redis diff --git a/README.md b/README.md index 2a07417..6770f66 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ - [Migrations](#migrations) +- [Messaging queue](#messaging-queue) + - [Code Walk-through](#code-walk-through) - [Config](#config) - [Command](#command) @@ -87,8 +89,78 @@ Migrations are like **version control for your database**, allowing your team to - **RUN :** To run migration there is two command `make migration-up` && `make migration-down`. - Migration needs `-- +migrate Up` and `-- +migrate Down` respectively in starting of files, this is required because we are using [sql-migrate](https://github.com/rubenv/sql-migrate) package - - +### **Messaging Queue** +- We are using [watermill](https://watermill.io/) package for messaging queue. +- Watermill is a Golang library for working efficiently with message streams. It is intended for building event-driven applications. + + +- #### Multiple Message Queue Broker Support + - We are supporting 3 types of message queue broker at this time `rabbitmq`, `redis` & `kafka` + - It allows us to switch to message queue broker without changing too much stuff. + - Watermill package allows us to do that. + - We have environment variable `MQ_DIALECT` where you can set to message queue broker type. + - Need to change env accoeding to message queue broker. +- #### Creating An Worker + - All of the workers for your application are stored in the `cli/workers` directory. + - To create a new job add a new file into the `cli/workers` directory. +- #### Class Structure + - Workers class are very simple, consisting of a single method `Handle`. `Handle` executes when a message is received. + - The `Handle()` method should return an `error` if the job fails. + ```go + type WelcomeMail struct { + FirstName string + LastName string + Email string + Roles string + } + // Handle executes the job. + func (w WelcomeMail) Handle() error { + return nil + } + ``` + - Command to run worker + ```go + go run app.go worker --retry-delay 400 --retry-count 3 --topic user + // --retry-delay 400 --retry-count 3 are optional + // --retry-delay 400 means it will retry after 400ms + // --retry-count 3 means it will retry 3 times + ``` +- #### Register Worker + - After creating the struct, you need to register it in `cli/workers/worker_handler.go`, so that it can be called correctly. + - To register a new worker add struct to `RegisterWorkerStruct` function. + ```go + func RegisterWorkerStruct() []interface{} { + return []interface{}{ + WelcomeMail{}, + // ... + } + } + ``` +- #### Publish Message + - The `InitPubliser` function initializes a `WatermillPubliser` based on the provided configuration. + ```go + pub, err := watermill.InitPubliser(cfg) + if err != nil { + // Handle error + } + ``` + - The `Publish` method on `WatermillPubliser` is used to publish a message to a specific topic(queue name). The message is encoded using the Go `encoding/gob` package before being sent. + ```go + // Worker struct must be registered before publishing + err := pub.Publish(topic, workerStruct) + if err != nil { + // Handle error + } + ``` +- #### Dead Letter Queue + - The `dead letter queue`, also known as the `poison queue` in watermill, is a designated destination for messages that have failed to undergo processing by a consumer. + - The name of this queue is specified in the `DEAD_LETTER_QUEUE` environment variable. + - Command to run dead letter queue + ```go + go run app.go dead-letter-queue + ``` + +--- ### **Code Walk-through** - #### Config: - We are using [envconfig](https://github.com/kelseyhightower/envconfig) which binds env values to struct elements. @@ -165,7 +237,7 @@ Migrations are like **version control for your database**, allowing your team to - Each controller must have their struct which contains model object, that will be use to call function of models. - #### Utils: - We have define some common methods inside `utils` like json response. - - We need common function which handles json response that's why we have created (json_response.go)[utils/json_response.go] + - We need common function which handles json response that's why we have created [json_response.go](utils/json_response.go) - Similarly we have created different files for different use in utils. --- ### **Testcases** diff --git a/cli/dead_letter_queue.go b/cli/dead_letter_queue.go index b238ebd..10baa12 100644 --- a/cli/dead_letter_queue.go +++ b/cli/dead_letter_queue.go @@ -12,7 +12,6 @@ import ( ) // GetAPICommandDef runs app -// ? need rename in file name or command name func GetDeadQueueCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { workerCommand := cobra.Command{ @@ -28,18 +27,15 @@ func GetDeadQueueCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Comm } // run worker with topic(queue name) and process function - if err := subscriber.Run(cfg.MQ.DeadQueue, HandleFailJob); err != nil { - return err - } - - return nil + err = subscriber.Run(cfg.MQ.DeadQueue, HandleFailJob) + return err }, } return workerCommand } func HandleFailJob(msg *message.Message) error { - fmt.Println("failed job", string(msg.Payload)) + fmt.Println("failed job:-", msg.UUID) // process here return nil } diff --git a/cli/main.go b/cli/main.go index f871a25..3fb0087 100644 --- a/cli/main.go +++ b/cli/main.go @@ -15,8 +15,8 @@ func Init(cfg config.AppConfig, logger *zap.Logger) error { workerCmd.PersistentFlags().Int("retry-count", 3, "number of retry") workerCmd.PersistentFlags().String("topic", "", "topic name(queue name)") - d:=GetDeadQueueCommandDef(cfg, logger) + deadQueueCmd:=GetDeadQueueCommandDef(cfg, logger) rootCmd := &cobra.Command{Use: "golang-api"} - rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd,&d) + rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd,&deadQueueCmd) return rootCmd.Execute() } diff --git a/cli/worker.go b/cli/worker.go index 55196b0..6bed939 100644 --- a/cli/worker.go +++ b/cli/worker.go @@ -34,7 +34,7 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command return err } - // Init worker + // Init subscriber subscriber, err := watermill.InitSubscriber(cfg) if err != nil { return err diff --git a/cli/workers/worker_handler.go b/cli/workers/worker_handler.go index 67ae2a5..c3a4e81 100644 --- a/cli/workers/worker_handler.go +++ b/cli/workers/worker_handler.go @@ -8,13 +8,13 @@ import ( ) func init() { - for _, v := range ListStruct() { + for _, v := range RegisterWorkerStruct() { gob.Register(v) } } // Register all worker struct here befour run worker for proper unmarshalling -func ListStruct() []interface{} { +func RegisterWorkerStruct() []interface{} { return []interface{}{ WelcomeMail{}, // ... diff --git a/config/mq.go b/config/mq.go index fb852f7..7873b95 100644 --- a/config/mq.go +++ b/config/mq.go @@ -1,6 +1,5 @@ package config -// TODO: add env to .env.example type MQConfig struct { Dialect string `envconfig:"MQ_DIALECT"` Debug bool `envconfig:"MQ_DEBUG"` diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index 512cdd4..4f3008d 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -107,12 +107,12 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) } - // publish job to queue - myEvent := workers.WelcomeMail{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Roles: userReq.Roles} - err = ctrl.pub.Publish("user", myEvent) + // publish message to queue + welcomeMail := workers.WelcomeMail{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Roles: userReq.Roles} + err = ctrl.pub.Publish("user", welcomeMail) if err != nil { - return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) + ctrl.logger.Error("error while publish message", zap.Error(err)) } - + return utils.JSONSuccess(c, http.StatusCreated, user) } diff --git a/pkg/watermill/publisher.go b/pkg/watermill/publisher.go index de29ea0..3b52245 100644 --- a/pkg/watermill/publisher.go +++ b/pkg/watermill/publisher.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/gob" "fmt" - "log" "github.com/Improwised/golang-api/cli/workers" "github.com/Improwised/golang-api/config" @@ -39,29 +38,31 @@ func InitPubliser(cfg config.AppConfig) (*WatermillPubliser, error) { } -// send message to queue using topic name +// send message into queue using topic name // // struct must from worker package(/cli/workers) -func (wp *WatermillPubliser) Publish(topic string, data interface{}) error { +func (wp *WatermillPubliser) Publish(topic string, inputStruct interface{}) error { + // if broker is not set then return nil + if wp.publisher == nil { + return nil + } var network bytes.Buffer enc := gob.NewEncoder(&network) var handle workers.Handler - handle, ok := data.(workers.Handler) + handle, ok := inputStruct.(workers.Handler) if !ok { - return fmt.Errorf("data is not of type workers.Handler") + return fmt.Errorf("struct is not of type workers.Handler") } err := enc.Encode(&handle) if err != nil { - log.Fatal("encode: niche", err) - } - msg := message.NewMessage(watermill.NewUUID(), network.Bytes()) - - if err := wp.publisher.Publish(topic, msg); err != nil { return err } - return nil + + msg := message.NewMessage(watermill.NewUUID(), network.Bytes()) + err = wp.publisher.Publish(topic, msg) + return err } func initAmqpPub(cfg config.AppConfig) (*WatermillPubliser, error) { From 5e24234358fabcec738590b48fc2be6f8048a858 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Thu, 25 Jan 2024 16:12:44 +0530 Subject: [PATCH 06/16] fix: remove unuse code --- cli/workers/user.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/workers/user.go b/cli/workers/user.go index 8006d3b..4ff5d37 100644 --- a/cli/workers/user.go +++ b/cli/workers/user.go @@ -16,12 +16,12 @@ type WelcomeMail struct { func (w WelcomeMail) Handle() error { log.Println("sending mail") - smtp := helpers.NewSMTPHelper("sandbox.smtp.mailtrap.io", "2525", "7628fa366c0257c", "2afb7200812272") + smtp := helpers.NewSMTPHelper("localhost", "2525", "root", "pass") smtp.SetSubject("welocme") smtp.SetPlainBody([]byte("welcome to our org")) - smtp.SetSender("chintansakhiya00001@gmail.com") - smtp.SetReceivers([]string{"chintansakhiya00001@gmail.com"}) + smtp.SetSender("support@improwised.com") + smtp.SetReceivers([]string{w.Email}) if err := smtp.SendMail(); err != nil { return err From 552af1f92767cd6796056d28f58c8f5b87535861 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Thu, 25 Jan 2024 16:21:50 +0530 Subject: [PATCH 07/16] fix: run pre-commit --- cli/main.go | 8 ++++---- controllers/api/v1/user_controller.go | 2 +- pkg/watermill/publisher.go | 1 - pkg/watermill/subscriber.go | 2 +- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cli/main.go b/cli/main.go index 3fb0087..0b4dba1 100644 --- a/cli/main.go +++ b/cli/main.go @@ -10,13 +10,13 @@ import ( func Init(cfg config.AppConfig, logger *zap.Logger) error { migrationCmd := GetMigrationCommandDef(cfg) apiCmd := GetAPICommandDef(cfg, logger) - workerCmd:=GetWorkerCommandDef(cfg, logger) + workerCmd := GetWorkerCommandDef(cfg, logger) workerCmd.PersistentFlags().Int("retry-delay", 100, "time intertval for two retry in ms") workerCmd.PersistentFlags().Int("retry-count", 3, "number of retry") workerCmd.PersistentFlags().String("topic", "", "topic name(queue name)") - - deadQueueCmd:=GetDeadQueueCommandDef(cfg, logger) + + deadQueueCmd := GetDeadQueueCommandDef(cfg, logger) rootCmd := &cobra.Command{Use: "golang-api"} - rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd,&deadQueueCmd) + rootCmd.AddCommand(&migrationCmd, &apiCmd, &workerCmd, &deadQueueCmd) return rootCmd.Execute() } diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index 4f3008d..43da013 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -106,7 +106,7 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { ctrl.logger.Error("error while insert user", zap.Error(err)) return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) } - + // publish message to queue welcomeMail := workers.WelcomeMail{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Roles: userReq.Roles} err = ctrl.pub.Publish("user", welcomeMail) diff --git a/pkg/watermill/publisher.go b/pkg/watermill/publisher.go index 3b52245..37857ba 100644 --- a/pkg/watermill/publisher.go +++ b/pkg/watermill/publisher.go @@ -97,4 +97,3 @@ func initKafkaPub(cfg config.AppConfig) (*WatermillPubliser, error) { ) return &WatermillPubliser{publisher: publisher}, err } - diff --git a/pkg/watermill/subscriber.go b/pkg/watermill/subscriber.go index 34f114e..d4c3bdf 100644 --- a/pkg/watermill/subscriber.go +++ b/pkg/watermill/subscriber.go @@ -92,7 +92,7 @@ func (ws *WatermillSubscriber) Run(topic string, handlerFunc message.NoPublishHa } ws.Router = router } - + ws.Router.AddNoPublisherHandler( "handler", topic, From 69f09d036cab38309aac74de7e23d869752759d9 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Thu, 25 Jan 2024 17:12:04 +0530 Subject: [PATCH 08/16] fix: added redis user name and password --- .env.example | 2 ++ config/mq.go | 2 ++ controllers/api/v1/user_controller.go | 12 ++++++------ pkg/watermill/publisher.go | 4 +++- pkg/watermill/subscriber.go | 7 +++---- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.env.example b/.env.example index ca8853a..98b7743 100644 --- a/.env.example +++ b/.env.example @@ -36,6 +36,8 @@ JWT_SECRET=ThisIsKey # MQ_DIALECT=redis # REDIS_HOST= # CONSUMER_GROUP= +# REDIS_USERNAME= +# REDIS_PASSWORD= # kafka # MQ_DIALECT=kafka diff --git a/config/mq.go b/config/mq.go index 7873b95..6107fcb 100644 --- a/config/mq.go +++ b/config/mq.go @@ -12,6 +12,8 @@ type MQConfig struct { type RedisConfig struct { RedisUrl string `envconfig:"REDIS_URI"` ConsumerGroup string `envconfig:"CONSUMER_GROUP"` + UserName string `envconfig:"REDIS_USERNAME"` + Password string `envconfig:"REDIS_PASSWORD"` } type AmqpConfig struct { diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index 43da013..2ed83e6 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -101,11 +101,11 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { return utils.JSONFail(c, http.StatusBadRequest, utils.ValidatorErrorString(err)) } - user, err := ctrl.userService.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, ctrl.event) - if err != nil { - ctrl.logger.Error("error while insert user", zap.Error(err)) - return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) - } + // user, err := ctrl.userService.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, ctrl.event) + // if err != nil { + // ctrl.logger.Error("error while insert user", zap.Error(err)) + // return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) + // } // publish message to queue welcomeMail := workers.WelcomeMail{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Roles: userReq.Roles} @@ -114,5 +114,5 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { ctrl.logger.Error("error while publish message", zap.Error(err)) } - return utils.JSONSuccess(c, http.StatusCreated, user) + return utils.JSONSuccess(c, http.StatusCreated, "user") } diff --git a/pkg/watermill/publisher.go b/pkg/watermill/publisher.go index 37857ba..5f7b862 100644 --- a/pkg/watermill/publisher.go +++ b/pkg/watermill/publisher.go @@ -71,10 +71,12 @@ func initAmqpPub(cfg config.AppConfig) (*WatermillPubliser, error) { return &WatermillPubliser{publisher: publisher}, err } -// TODO: username/pass + func initRedisPub(cfg config.AppConfig) (*WatermillPubliser, error) { pubClient := redis.NewClient(&redis.Options{ Addr: cfg.MQ.Redis.RedisUrl, + Username: cfg.MQ.Redis.UserName, + Password: cfg.MQ.Redis.Password, }) publisher, err := redisstream.NewPublisher( redisstream.PublisherConfig{ diff --git a/pkg/watermill/subscriber.go b/pkg/watermill/subscriber.go index d4c3bdf..3e1ee13 100644 --- a/pkg/watermill/subscriber.go +++ b/pkg/watermill/subscriber.go @@ -26,8 +26,7 @@ type WatermillSubscriber struct { Router *message.Router } -// TODO: for redis and kafka username/password -// TODO: add other dialect + func InitSubscriber(cfg config.AppConfig) (*WatermillSubscriber, error) { logger = watermill.NewStdLogger(cfg.MQ.Debug, cfg.MQ.Track) switch cfg.MQ.Dialect { @@ -134,8 +133,8 @@ func initKafkaSub(cfg config.AppConfig) (*WatermillSubscriber, error) { func initRedisSub(cfg config.AppConfig) (*WatermillSubscriber, error) { subClient := redis.NewClient(&redis.Options{ Addr: cfg.MQ.Redis.RedisUrl, - // Username: , - // password: , + Username: cfg.MQ.Redis.UserName, + Password: cfg.MQ.Redis.Password, }) subscriber, err := redisstream.NewSubscriber( redisstream.SubscriberConfig{ From 89ecfa5aac2d52819378f65d652c3aadc871ab58 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Thu, 25 Jan 2024 19:13:11 +0530 Subject: [PATCH 09/16] fix: api response --- .env.example | 2 +- controllers/api/v1/user_controller.go | 12 ++++++------ go.mod | 1 + go.sum | 4 ++-- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index 98b7743..b06e099 100644 --- a/.env.example +++ b/.env.example @@ -6,7 +6,7 @@ APP_ENV=local # Database Config DB_DIALECT=postgres DB_HOST=localhost -DB_PORT=26257 +DB_PORT=5432 DB_USERNAME=golang-api DB_PASSWORD=golang-api DB_NAME=golang-api diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index 2ed83e6..43da013 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -101,11 +101,11 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { return utils.JSONFail(c, http.StatusBadRequest, utils.ValidatorErrorString(err)) } - // user, err := ctrl.userService.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, ctrl.event) - // if err != nil { - // ctrl.logger.Error("error while insert user", zap.Error(err)) - // return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) - // } + user, err := ctrl.userService.RegisterUser(models.User{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Password: userReq.Password, Roles: userReq.Roles}, ctrl.event) + if err != nil { + ctrl.logger.Error("error while insert user", zap.Error(err)) + return utils.JSONError(c, http.StatusInternalServerError, constants.ErrInsertUser) + } // publish message to queue welcomeMail := workers.WelcomeMail{FirstName: userReq.FirstName, LastName: userReq.LastName, Email: userReq.Email, Roles: userReq.Roles} @@ -114,5 +114,5 @@ func (ctrl *UserController) CreateUser(c *fiber.Ctx) error { ctrl.logger.Error("error while publish message", zap.Error(err)) } - return utils.JSONSuccess(c, http.StatusCreated, "user") + return utils.JSONSuccess(c, http.StatusCreated, user) } diff --git a/go.mod b/go.mod index 5bccdaf..20a40a2 100644 --- a/go.mod +++ b/go.mod @@ -91,6 +91,7 @@ require ( github.com/rabbitmq/amqp091-go v1.9.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.4.4 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index c56695b..6ec5b84 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rubenv/sql-migrate v1.5.2 h1:bMDqOnrJVV/6JQgQ/MxOpU+AdO8uzYYA/TxFUBzFtS0= From 14c2a311f5f4a45e160c60878088ff75fe829205 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Mon, 29 Jan 2024 10:35:42 +0530 Subject: [PATCH 10/16] doc: update an readme file --- README.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 6770f66..6431c18 100644 --- a/README.md +++ b/README.md @@ -118,13 +118,6 @@ Migrations are like **version control for your database**, allowing your team to return nil } ``` - - Command to run worker - ```go - go run app.go worker --retry-delay 400 --retry-count 3 --topic user - // --retry-delay 400 --retry-count 3 are optional - // --retry-delay 400 means it will retry after 400ms - // --retry-count 3 means it will retry 3 times - ``` - #### Register Worker - After creating the struct, you need to register it in `cli/workers/worker_handler.go`, so that it can be called correctly. - To register a new worker add struct to `RegisterWorkerStruct` function. @@ -136,6 +129,15 @@ Migrations are like **version control for your database**, allowing your team to } } ``` + - #### Command to run worker + ```go + go run app.go worker --retry-delay 400 --retry-count 3 --topic user + // --retry-delay 400 --retry-count 3 are optional + // --retry-delay 400 means it will retry after 400ms + // --retry-count 3 means it will retry 3 times + + ``` + - #### Publish Message - The `InitPubliser` function initializes a `WatermillPubliser` based on the provided configuration. ```go From d83ec0e04506f9d1973b76318999c1b4ebc38c7d Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Mon, 29 Jan 2024 15:52:54 +0530 Subject: [PATCH 11/16] feat: added google cloud broker --- .env.example | 5 ++ README.md | 2 +- config/mq.go | 24 ++++--- go.mod | 34 ++++++++-- go.sum | 126 ++++++++++++++++++++++++++++++++++++ pkg/watermill/publisher.go | 17 ++++- pkg/watermill/subscriber.go | 37 ++++++++++- 7 files changed, 226 insertions(+), 19 deletions(-) diff --git a/.env.example b/.env.example index b06e099..d972cc4 100644 --- a/.env.example +++ b/.env.example @@ -43,3 +43,8 @@ JWT_SECRET=ThisIsKey # MQ_DIALECT=kafka # KAFKA_BROKER= # CONSUMER_GROUP= + +# googleCloud +# MQ_DIALECT=googlecloud +# PROJECT_ID= +# SUBSCRIPTION_ID= diff --git a/README.md b/README.md index 6431c18..e0626d3 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Migrations are like **version control for your database**, allowing your team to - #### Multiple Message Queue Broker Support - - We are supporting 3 types of message queue broker at this time `rabbitmq`, `redis` & `kafka` + - We are supporting 4 types of message queue broker at this time `rabbitmq`, `redis`, `googleCloud` & `kafka` - It allows us to switch to message queue broker without changing too much stuff. - Watermill package allows us to do that. - We have environment variable `MQ_DIALECT` where you can set to message queue broker type. diff --git a/config/mq.go b/config/mq.go index 6107fcb..d9b8155 100644 --- a/config/mq.go +++ b/config/mq.go @@ -1,19 +1,20 @@ package config type MQConfig struct { - Dialect string `envconfig:"MQ_DIALECT"` - Debug bool `envconfig:"MQ_DEBUG"` - Track bool `envconfig:"MQ_TRACK"` - DeadQueue string `envconfig:"DEAD_LETTER_QUEUE"` - Redis RedisConfig - Amqp AmqpConfig - Kafka KafkaConfig + Dialect string `envconfig:"MQ_DIALECT"` + Debug bool `envconfig:"MQ_DEBUG"` + Track bool `envconfig:"MQ_TRACK"` + DeadQueue string `envconfig:"DEAD_LETTER_QUEUE"` + Redis RedisConfig + Amqp AmqpConfig + Kafka KafkaConfig + GoogleCloud GoogleCloud } type RedisConfig struct { RedisUrl string `envconfig:"REDIS_URI"` ConsumerGroup string `envconfig:"CONSUMER_GROUP"` - UserName string `envconfig:"REDIS_USERNAME"` - Password string `envconfig:"REDIS_PASSWORD"` + UserName string `envconfig:"REDIS_USERNAME"` + Password string `envconfig:"REDIS_PASSWORD"` } type AmqpConfig struct { @@ -23,3 +24,8 @@ type KafkaConfig struct { KafkaBroker []string `envconfig:"KAFKA_BROKER"` ConsumerGroup string `envconfig:"CONSUMER_GROUP"` } + +type GoogleCloud struct { + ProjectID string `envconfig:"PROJECT_ID"` + SubscriptionId string `envconfig:"SUBSCRIPTION_ID"` +} diff --git a/go.mod b/go.mod index 20a40a2..41d5bd1 100644 --- a/go.mod +++ b/go.mod @@ -6,10 +6,12 @@ toolchain go1.21.5 require ( clevergo.tech/jsend v1.1.3 + cloud.google.com/go/pubsub v1.36.0 github.com/Shopify/sarama v1.38.0 github.com/ThreeDotsLabs/watermill v1.3.5 github.com/ThreeDotsLabs/watermill-amqp v1.1.4 github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.1 + github.com/ThreeDotsLabs/watermill-googlecloud v1.1.0 github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0 github.com/ThreeDotsLabs/watermill-redisstream v1.2.2 github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef @@ -40,6 +42,10 @@ require ( require golang.org/x/sync v0.6.0 // indirect require ( + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.5 // indirect github.com/Rican7/retry v0.3.1 // indirect github.com/andybalholm/brotli v1.0.5 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -51,15 +57,20 @@ require ( github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.5.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -101,17 +112,28 @@ require ( github.com/valyala/fasthttp v1.48.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.31.0 // indirect - go.opentelemetry.io/otel v1.6.1 // indirect - go.opentelemetry.io/otel/trace v1.6.1 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/api v0.157.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac // indirect + google.golang.org/grpc v1.60.1 // indirect + google.golang.org/protobuf v1.32.0 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 6ec5b84..9976686 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,16 @@ clevergo.tech/jsend v1.1.3 h1:noSA5WtIrEfX4gKxlJB/EQTpbHUxkK2E+nR9pguMZsI= clevergo.tech/jsend v1.1.3/go.mod h1:0w6SXsvj2f62Dy8fHBHFrMWQMB5K2uIzfiDFIMFh82k= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/pubsub v1.36.0 h1:cgaJ0mgwEM0YNNATXFXSnLfji6XcMieTc2mRjH1ZYdY= +cloud.google.com/go/pubsub v1.36.0/go.mod h1:qQvGW4ANjuYcOpTMTy5+u6HBIoJF00cPfQ/ubMcc/D8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= @@ -17,6 +28,8 @@ github.com/ThreeDotsLabs/watermill-amqp v1.1.4 h1:vOdc8a0m0sMPAJZ2CMLx5a+fwlgeeo github.com/ThreeDotsLabs/watermill-amqp v1.1.4/go.mod h1:5RtpKNTriXCWQZ67YDg1G7qsphZoUue/EWOmQqTZi3Q= github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.1 h1:OxvMB2/3YtcQuC7quC+CGmFpGz9oaxP2ef5wkp+R2oM= github.com/ThreeDotsLabs/watermill-amqp/v2 v2.1.1/go.mod h1:MCNoh0HUg4w0bY64on9BnhUodHeimz8+vMfXrzyuWN8= +github.com/ThreeDotsLabs/watermill-googlecloud v1.1.0 h1:3Y/Wo+pHNnViKHIlBgpDl9LhxUXN9/jtJ+yrbyypvSQ= +github.com/ThreeDotsLabs/watermill-googlecloud v1.1.0/go.mod h1:cLifbk8zyGZ4MEqCmjE5vyTQOnTmy0kZUlrMzoixBBk= github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0 h1:/KYEjLlLx6nW3jn6AEcwAlWkPWP62zi/sUsEP4uKkZE= github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0/go.mod h1:w+9jhI7x5ZP67ceSUIIpkgLzjAakotfHX4sWyqsKVjs= github.com/ThreeDotsLabs/watermill-redisstream v1.2.2 h1:/fFHagJiObMBbYIDrygRoAq+RxqLPcQZdGi6b0ViG08= @@ -35,8 +48,11 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -58,6 +74,12 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8 github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= @@ -70,6 +92,8 @@ github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpj github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -94,21 +118,50 @@ github.com/gofiber/adaptor/v2 v2.2.1/go.mod h1:AhR16dEqs25W2FY/l8gSj1b51Azg5dtPD github.com/gofiber/fiber/v2 v2.48.0 h1:cRVMCb9aUJDsyHxGFLwz/sGzDggdailZZyptU9F9cU0= github.com/gofiber/fiber/v2 v2.48.0/go.mod h1:xqJgfqrc23FJuqGOW6DVgi3HyZEm2Mn9pRqUb2kHSX8= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= @@ -207,6 +260,7 @@ github.com/poy/onpar v1.1.2 h1:QaNrNiZx0+Nar5dLgTVp5mXkyoVFIbepjyEoGSnhbAY= github.com/poy/onpar v1.1.2/go.mod h1:6X8FLNoxyr9kkmnlqpK6LSoiOtrO6MICtWwEuWkLjzg= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= @@ -259,6 +313,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= @@ -274,12 +329,24 @@ github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.31.0 h1:J8jI81RCB7U9a3qsTZXM/38XrvbLJCye6J32bfQctYY= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.31.0/go.mod h1:72+cPzsW6geApbceSLMbZtYZeGMgtRDw5TcSEsdGlhc= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v1.6.1 h1:6r1YrcTenBvYa1x491d0GGpTVBsNECmrc/K6b+zDeis= go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/trace v1.6.1 h1:f8c93l5tboBYZna1nWk0W9DYyMzJXDWdZcJZ0Kb400U= go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= @@ -291,6 +358,7 @@ go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -298,14 +366,23 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211029224645-99673261e6eb/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -316,15 +393,22 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -351,23 +435,63 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20= +google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457 h1:KHBtwE+eQc3+NxpjmRFlQ3pJQ2FNnhhgB9xOV8kyBuU= +google.golang.org/genproto/googleapis/api v0.0.0-20240122161410-6c6643bf1457/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac h1:nUQEQmH/csSvFECKYRv6HWEyypysidKl2I6Qpsglq/0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -384,3 +508,5 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/watermill/publisher.go b/pkg/watermill/publisher.go index 5f7b862..ca51466 100644 --- a/pkg/watermill/publisher.go +++ b/pkg/watermill/publisher.go @@ -4,11 +4,13 @@ import ( "bytes" "encoding/gob" "fmt" + "time" "github.com/Improwised/golang-api/cli/workers" "github.com/Improwised/golang-api/config" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amqp/v2/pkg/amqp" + "github.com/ThreeDotsLabs/watermill-googlecloud/pkg/googlecloud" "github.com/ThreeDotsLabs/watermill-kafka/v2/pkg/kafka" "github.com/ThreeDotsLabs/watermill-redisstream/pkg/redisstream" @@ -32,6 +34,8 @@ func InitPubliser(cfg config.AppConfig) (*WatermillPubliser, error) { case "kafka": return initKafkaPub(cfg) + case "googlecloud": + return initGoogleCloudPub(cfg) default: return &WatermillPubliser{}, nil } @@ -74,7 +78,7 @@ func initAmqpPub(cfg config.AppConfig) (*WatermillPubliser, error) { func initRedisPub(cfg config.AppConfig) (*WatermillPubliser, error) { pubClient := redis.NewClient(&redis.Options{ - Addr: cfg.MQ.Redis.RedisUrl, + Addr: cfg.MQ.Redis.RedisUrl, Username: cfg.MQ.Redis.UserName, Password: cfg.MQ.Redis.Password, }) @@ -99,3 +103,14 @@ func initKafkaPub(cfg config.AppConfig) (*WatermillPubliser, error) { ) return &WatermillPubliser{publisher: publisher}, err } + +func initGoogleCloudPub(cfg config.AppConfig) (*WatermillPubliser, error) { + publisher, err := googlecloud.NewPublisher(googlecloud.PublisherConfig{ + ProjectID: cfg.MQ.GoogleCloud.ProjectID, + ConnectTimeout: 10 * time.Second, + PublishTimeout: 10 * time.Second, + Marshaler: googlecloud.DefaultMarshalerUnmarshaler{}, + }, logger) + + return &WatermillPubliser{publisher: publisher}, err +} diff --git a/pkg/watermill/subscriber.go b/pkg/watermill/subscriber.go index 3e1ee13..f347217 100644 --- a/pkg/watermill/subscriber.go +++ b/pkg/watermill/subscriber.go @@ -5,12 +5,14 @@ import ( "fmt" "time" + "cloud.google.com/go/pubsub" "github.com/Improwised/golang-api/config" "github.com/Shopify/sarama" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amqp/pkg/amqp" + "github.com/ThreeDotsLabs/watermill-googlecloud/pkg/googlecloud" "github.com/ThreeDotsLabs/watermill-kafka/v2/pkg/kafka" "github.com/ThreeDotsLabs/watermill-redisstream/pkg/redisstream" "github.com/ThreeDotsLabs/watermill/message" @@ -26,7 +28,6 @@ type WatermillSubscriber struct { Router *message.Router } - func InitSubscriber(cfg config.AppConfig) (*WatermillSubscriber, error) { logger = watermill.NewStdLogger(cfg.MQ.Debug, cfg.MQ.Track) switch cfg.MQ.Dialect { @@ -39,6 +40,9 @@ func InitSubscriber(cfg config.AppConfig) (*WatermillSubscriber, error) { case "kafka": return initKafkaSub(cfg) + case "googlecloud": + return initGoogleCloudSub(cfg) + default: return nil, nil } @@ -132,7 +136,7 @@ func initKafkaSub(cfg config.AppConfig) (*WatermillSubscriber, error) { func initRedisSub(cfg config.AppConfig) (*WatermillSubscriber, error) { subClient := redis.NewClient(&redis.Options{ - Addr: cfg.MQ.Redis.RedisUrl, + Addr: cfg.MQ.Redis.RedisUrl, Username: cfg.MQ.Redis.UserName, Password: cfg.MQ.Redis.Password, }) @@ -146,3 +150,32 @@ func initRedisSub(cfg config.AppConfig) (*WatermillSubscriber, error) { ) return &WatermillSubscriber{Subscriber: subscriber}, err } + +func initGoogleCloudSub(cfg config.AppConfig) (*WatermillSubscriber, error) { + subscriptionName := func(string) string { + return cfg.MQ.GoogleCloud.SubscriptionId + } + ackDeadline := 20 * time.Second + subscriber, err := googlecloud.NewSubscriber( + googlecloud.SubscriberConfig{ + ProjectID: cfg.MQ.GoogleCloud.ProjectID, + DoNotCreateTopicIfMissing: false, + DoNotCreateSubscriptionIfMissing: false, + InitializeTimeout: 30 * time.Second, + GenerateSubscriptionName: subscriptionName, + SubscriptionConfig: pubsub.SubscriptionConfig{ + RetainAckedMessages: false, + EnableMessageOrdering: false, + AckDeadline: ackDeadline, + RetentionDuration: 24 * time.Hour, + }, + Unmarshaler: googlecloud.DefaultMarshalerUnmarshaler{}, + }, + logger, + ) + if err != nil { + panic(err) + } + + return &WatermillSubscriber{Subscriber: subscriber}, err +} From 59abc6561ba8595cdc4cba76c16f73fe4b26a139 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Mon, 29 Jan 2024 16:05:45 +0530 Subject: [PATCH 12/16] fix: go.mod verson --- go.mod | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 41d5bd1..cded92d 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/Improwised/golang-api -go 1.21 - -toolchain go1.21.5 +go 1.19 require ( clevergo.tech/jsend v1.1.3 From 9d6ff002ad5aa08565520a2b39dcedad0ab30676 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Tue, 30 Jan 2024 19:28:18 +0530 Subject: [PATCH 13/16] feat: added sql broker --- .env.example | 17 ++- README.md | 4 +- cli/api.go | 2 +- cli/dead_letter_queue.go | 32 ++++- cli/worker.go | 6 +- cli/workers/worker_handler.go | 6 + config/mq.go | 14 ++- controllers/api/v1/user_controller.go | 4 +- database/watermill_mysql_schema.go | 115 ++++++++++++++++++ database/watermill_postgres_schema.go | 167 ++++++++++++++++++++++++++ go.mod | 8 +- go.sum | 45 ++++--- pkg/watermill/publisher.go | 90 +++++++++++--- pkg/watermill/subscriber.go | 71 ++++++++++- routes/main.go | 4 +- 15 files changed, 530 insertions(+), 55 deletions(-) create mode 100644 database/watermill_mysql_schema.go create mode 100644 database/watermill_postgres_schema.go diff --git a/.env.example b/.env.example index d972cc4..b33489b 100644 --- a/.env.example +++ b/.env.example @@ -24,9 +24,20 @@ SQLITE_FILEPATH=database/golang-api.db JWT_SECRET=ThisIsKey # Message queue config -# MQ_TRACK=true -# MQ_DEBUG=true -# DEAD_LETTER_QUEUE=dead_queue +MQ_TRACK=true +MQ_DEBUG=true +DEAD_LETTER_QUEUE=dead_queue +HANDLER_NAME=handler + +# sql +MQ_DIALECT=sql +MQ_DB_DIALECT=postgres +MQ_DB_HOST=localhost +MQ_DB_PORT=5432 +MQ_DB_USERNAME=golang-api +MQ_DB_PASSWORD=golang-api +MQ_DB_NAME=golang-api +MQ_DB_QUERYSTRING=sslmode=disable # Rabbitmq # MQ_DIALECT=amqp diff --git a/README.md b/README.md index e0626d3..f1fe7c7 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Migrations are like **version control for your database**, allowing your team to - #### Multiple Message Queue Broker Support - - We are supporting 4 types of message queue broker at this time `rabbitmq`, `redis`, `googleCloud` & `kafka` + - We are supporting 5 types of message queue broker at this time `rabbitmq`, `redis`, `googleCloud`,`sql(postges,mysql)` & `kafka` - It allows us to switch to message queue broker without changing too much stuff. - Watermill package allows us to do that. - We have environment variable `MQ_DIALECT` where you can set to message queue broker type. @@ -156,7 +156,7 @@ Migrations are like **version control for your database**, allowing your team to ``` - #### Dead Letter Queue - The `dead letter queue`, also known as the `poison queue` in watermill, is a designated destination for messages that have failed to undergo processing by a consumer. - - The name of this queue is specified in the `DEAD_LETTER_QUEUE` environment variable. + - The name of this queue is specified in the `DEAD_LETTER_QUEUE` environment variable, we are storing failed job into database. - Command to run dead letter queue ```go go run app.go dead-letter-queue diff --git a/cli/api.go b/cli/api.go index 2216b3c..d8bfacf 100644 --- a/cli/api.go +++ b/cli/api.go @@ -44,7 +44,7 @@ func GetAPICommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { return err } - pub, err := watermill.InitPubliser(cfg) + pub, err := watermill.InitPublisher(cfg,false) if err != nil { return err } diff --git a/cli/dead_letter_queue.go b/cli/dead_letter_queue.go index 10baa12..3c163a9 100644 --- a/cli/dead_letter_queue.go +++ b/cli/dead_letter_queue.go @@ -1,7 +1,8 @@ package cli import ( - "fmt" + "encoding/json" + "log" "go.uber.org/zap" @@ -11,6 +12,13 @@ import ( "github.com/spf13/cobra" ) +type DeadLetterQ struct { + Handler string `json:"handler_poisoned"` + Reason string `json:"reason_poisoned"` + Subscriber string `json:"subscriber_poisoned"` + Topic string `json:"topic_poisoned"` +} + // GetAPICommandDef runs app func GetDeadQueueCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command { @@ -21,13 +29,13 @@ func GetDeadQueueCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Comm RunE: func(cmd *cobra.Command, args []string) error { // Init worker - subscriber, err := watermill.InitSubscriber(cfg) + subscriber, err := watermill.InitSubscriber(cfg, true) if err != nil { return err } // run worker with topic(queue name) and process function - err = subscriber.Run(cfg.MQ.DeadQueue, HandleFailJob) + err = subscriber.Run(cfg.MQ.DeadLetterQ,"dead_letter_queue" ,HandleFailJob) return err }, } @@ -35,7 +43,21 @@ func GetDeadQueueCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Comm } func HandleFailJob(msg *message.Message) error { - fmt.Println("failed job:-", msg.UUID) - // process here + // get fail job details from metadata + var result DeadLetterQ + metadata := make(map[string]string) + for k, v := range msg.Metadata { + metadata[k] = v + } + jsonString, err := json.Marshal(metadata) + if err != nil { + return err + } + err = json.Unmarshal(jsonString, &result) + if err != nil { + return err + } + + log.Printf("Failed job handler: %s, reason: %s, subscriber: %s, topic: %s", result.Handler, result.Reason, result.Subscriber, result.Topic) return nil } diff --git a/cli/worker.go b/cli/worker.go index 6bed939..00f41ae 100644 --- a/cli/worker.go +++ b/cli/worker.go @@ -18,7 +18,7 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command Short: "To start worker", Long: `To start worker`, RunE: func(cmd *cobra.Command, args []string) error { - // Get details name from flag + // Get details from flag topic, err := cmd.Flags().GetString("topic") if err != nil { return err @@ -35,7 +35,7 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command } // Init subscriber - subscriber, err := watermill.InitSubscriber(cfg) + subscriber, err := watermill.InitSubscriber(cfg, false) if err != nil { return err } @@ -47,7 +47,7 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command } // run worker with topic(queue name) and process function - err = router.Run(topic, workers.Process) + err = router.Run(topic, cfg.MQ.HandlerName, workers.Process) return err }, diff --git a/cli/workers/worker_handler.go b/cli/workers/worker_handler.go index c3a4e81..ab430aa 100644 --- a/cli/workers/worker_handler.go +++ b/cli/workers/worker_handler.go @@ -3,6 +3,7 @@ package workers import ( "bytes" "encoding/gob" + "encoding/json" "github.com/ThreeDotsLabs/watermill/message" ) @@ -36,6 +37,11 @@ func Process(msg *message.Message) error { if err != nil { return err } + // Store the JSON payload in the msg so that it can be persisted in the database in case the job fails. + msg.Payload, err = json.Marshal(result) + if err != nil { + return err + } if err := result.Handle(); err != nil { return err } diff --git a/config/mq.go b/config/mq.go index d9b8155..6d55fb7 100644 --- a/config/mq.go +++ b/config/mq.go @@ -4,11 +4,13 @@ type MQConfig struct { Dialect string `envconfig:"MQ_DIALECT"` Debug bool `envconfig:"MQ_DEBUG"` Track bool `envconfig:"MQ_TRACK"` - DeadQueue string `envconfig:"DEAD_LETTER_QUEUE"` + DeadLetterQ string `envconfig:"DEAD_LETTER_QUEUE"` + HandlerName string `envconfig:"HANDLER_NAME"` Redis RedisConfig Amqp AmqpConfig Kafka KafkaConfig GoogleCloud GoogleCloud + Sql Sql } type RedisConfig struct { RedisUrl string `envconfig:"REDIS_URI"` @@ -29,3 +31,13 @@ type GoogleCloud struct { ProjectID string `envconfig:"PROJECT_ID"` SubscriptionId string `envconfig:"SUBSCRIPTION_ID"` } + +type Sql struct { + Dialect string `envconfig:"MQ_DB_DIALECT"` + Host string `envconfig:"MQ_DB_HOST"` + Port int `envconfig:"MQ_DB_PORT"` + Username string `envconfig:"MQ_DB_USERNAME"` + Password string `envconfig:"MQ_DB_PASSWORD"` + Db string `envconfig:"MQ_DB_NAME"` + QueryString string `envconfig:"DB_QUERYSTRING"` +} diff --git a/controllers/api/v1/user_controller.go b/controllers/api/v1/user_controller.go index 43da013..df113eb 100644 --- a/controllers/api/v1/user_controller.go +++ b/controllers/api/v1/user_controller.go @@ -25,11 +25,11 @@ type UserController struct { userService *services.UserService logger *zap.Logger event *events.Events - pub *watermill.WatermillPubliser + pub *watermill.WatermillPublisher } // NewUserController returns a user -func NewUserController(goqu *goqu.Database, logger *zap.Logger, event *events.Events, pub *watermill.WatermillPubliser) (*UserController, error) { +func NewUserController(goqu *goqu.Database, logger *zap.Logger, event *events.Events, pub *watermill.WatermillPublisher) (*UserController, error) { userModel, err := models.InitUserModel(goqu) if err != nil { return nil, err diff --git a/database/watermill_mysql_schema.go b/database/watermill_mysql_schema.go new file mode 100644 index 0000000..703bf98 --- /dev/null +++ b/database/watermill_mysql_schema.go @@ -0,0 +1,115 @@ +package database + +import ( + stdSQL "database/sql" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/Improwised/golang-api/config" + "github.com/ThreeDotsLabs/watermill-sql/v2/pkg/sql" + + "github.com/ThreeDotsLabs/watermill/message" + "github.com/pkg/errors" +) + +// custom schema for mysql database +// source: https://github.com/ThreeDotsLabs/watermill-sql/blob/master/pkg/sql/schema_adapter_mysql.go +type MySQLSchema struct { + GenerateMessagesTableName func(topic string) string + SubscribeBatchSize int +} +func (s MySQLSchema) SchemaInitializingQueries(topic string) []string { + createMessagesTable := strings.Join([]string{ + "CREATE TABLE IF NOT EXISTS " + s.MessagesTable(topic) + " (", + "`offset` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,", + "`uuid` VARCHAR(36) NOT NULL,", + "`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,", + "`payload` BINARY(255) NULL,", + "`metadata` JSON DEFAULT NULL", + ");", + }, "\n") + + return []string{createMessagesTable} +} + +func (s MySQLSchema) InsertQuery(topic string, msgs message.Messages) (string, []interface{}, error) { + insertQuery := fmt.Sprintf( + `INSERT INTO %s (uuid, payload, metadata) VALUES %s`, + s.MessagesTable(topic), + strings.TrimRight(strings.Repeat(`(?,?,?),`, len(msgs)), ","), + ) + + args, err := defaultInsertArgs(msgs) + if err != nil { + return "", nil, err + } + + return insertQuery, args, nil +} + +func (s MySQLSchema) batchSize() int { + if s.SubscribeBatchSize == 0 { + return 100 + } + + return s.SubscribeBatchSize +} + +func (s MySQLSchema) SelectQuery(topic string, consumerGroup string, offsetsAdapter sql.OffsetsAdapter) (string, []interface{}) { + nextOffsetQuery, nextOffsetArgs := offsetsAdapter.NextOffsetQuery(topic, consumerGroup) + selectQuery := ` + SELECT offset, uuid, payload, metadata FROM ` + s.MessagesTable(topic) + ` + WHERE + offset > (` + nextOffsetQuery + `) + ORDER BY + offset ASC + LIMIT ` + fmt.Sprintf("%d", s.batchSize()) + + return selectQuery, nextOffsetArgs +} + +func (s MySQLSchema) UnmarshalMessage(row sql.Scanner) (sql.Row, error) { + r := sql.Row{} + err := row.Scan(&r.Offset, &r.UUID, &r.Payload, &r.Metadata) + if err != nil { + return sql.Row{}, errors.Wrap(err, "could not scan message row") + } + + msg := message.NewMessage(string(r.UUID), r.Payload) + + if r.Metadata != nil { + err = json.Unmarshal(r.Metadata, &msg.Metadata) + if err != nil { + return sql.Row{}, errors.Wrap(err, "could not unmarshal metadata as JSON") + } + } + + r.Msg = msg + + return r, nil +} + +func (s MySQLSchema) MessagesTable(topic string) string { + if s.GenerateMessagesTableName != nil { + return s.GenerateMessagesTableName(topic) + } + return fmt.Sprintf("`watermill_%s`", topic) +} + +func (s MySQLSchema) SubscribeIsolationLevel() stdSQL.IsolationLevel { + // MySQL requires serializable isolation level for not losing messages. + return stdSQL.LevelSerializable +} + +func MysqlDBConnection(cfg config.Sql) (*stdSQL.DB, error) { + dbURL := cfg.Username + ":" + cfg.Password + "@tcp(" + cfg.Host + ":" + strconv.Itoa(cfg.Port) + ")/" + cfg.Db + + db, err := stdSQL.Open("mysql", dbURL) + if err != nil { + return nil, err + } + + return db, err +} diff --git a/database/watermill_postgres_schema.go b/database/watermill_postgres_schema.go new file mode 100644 index 0000000..ca8c95c --- /dev/null +++ b/database/watermill_postgres_schema.go @@ -0,0 +1,167 @@ +package database + +import ( + stdSQL "database/sql" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/Improwised/golang-api/config" + "github.com/ThreeDotsLabs/watermill-sql/v2/pkg/sql" + "github.com/pkg/errors" + + "github.com/ThreeDotsLabs/watermill/message" +) + +// custom schema for postgres database +// source: https://github.com/ThreeDotsLabs/watermill-sql/blob/master/pkg/sql/schema_adapter_postgresql.go + +type PostgreSQLSchema struct { + GenerateMessagesTableName func(topic string) string + SubscribeBatchSize int +} + +func (s PostgreSQLSchema) SchemaInitializingQueries(topic string) []string { + createMessagesTable := ` + CREATE TABLE IF NOT EXISTS ` + s.MessagesTable(topic) + ` ( + "offset" SERIAL, + "uuid" VARCHAR(36) NOT NULL, + "created_at" TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + "payload" bytea DEFAULT NULL, + "metadata" JSON DEFAULT NULL, + "transaction_id" xid8 NOT NULL, + PRIMARY KEY ("transaction_id", "offset") + ); + ` + + return []string{createMessagesTable} +} +func defaultInsertArgs(msgs message.Messages) ([]interface{}, error) { + var args []interface{} + for _, msg := range msgs { + metadata, err := json.Marshal(msg.Metadata) + if err != nil { + return nil, errors.Wrapf(err, "could not marshal metadata into JSON for message %s", msg.UUID) + } + + args = append(args, msg.UUID, msg.Payload, metadata) + } + + return args, nil +} + +func (s PostgreSQLSchema) InsertQuery(topic string, msgs message.Messages) (string, []interface{}, error) { + insertQuery := fmt.Sprintf( + `INSERT INTO %s (uuid, payload, metadata, transaction_id) VALUES %s`, + s.MessagesTable(topic), + defaultInsertMarkers(len(msgs)), + ) + + args, err := defaultInsertArgs(msgs) + if err != nil { + return "", nil, err + } + // log.Fatal("insertQuery", args) + return insertQuery, args, nil +} + +func defaultInsertMarkers(count int) string { + result := strings.Builder{} + + index := 1 + for i := 0; i < count; i++ { + result.WriteString(fmt.Sprintf("($%d,$%d,$%d,pg_current_xact_id()),", index, index+1, index+2)) + index += 3 + } + + return strings.TrimRight(result.String(), ",") +} + +func (s PostgreSQLSchema) batchSize() int { + if s.SubscribeBatchSize == 0 { + return 100 + } + + return s.SubscribeBatchSize +} + +func (s PostgreSQLSchema) SelectQuery(topic string, consumerGroup string, offsetsAdapter sql.OffsetsAdapter) (string, []interface{}) { + // Query inspired by https://event-driven.io/en/ordering_in_postgres_outbox/ + + nextOffsetQuery, nextOffsetArgs := offsetsAdapter.NextOffsetQuery(topic, consumerGroup) + selectQuery := ` + WITH last_processed AS ( + ` + nextOffsetQuery + ` + ) + + SELECT "offset", transaction_id, uuid, payload, metadata FROM ` + s.MessagesTable(topic) + ` + + WHERE + ( + ( + transaction_id = (SELECT last_processed_transaction_id FROM last_processed) + AND + "offset" > (SELECT offset_acked FROM last_processed) + ) + OR + (transaction_id > (SELECT last_processed_transaction_id FROM last_processed)) + ) + AND + transaction_id < pg_snapshot_xmin(pg_current_snapshot()) + ORDER BY + transaction_id ASC, + "offset" ASC + LIMIT ` + fmt.Sprintf("%d", s.batchSize()) + + return selectQuery, nextOffsetArgs +} + +func (s PostgreSQLSchema) UnmarshalMessage(row sql.Scanner) (sql.Row, error) { + r := sql.Row{} + var transactionID int64 + + err := row.Scan(&r.Offset, &transactionID, &r.UUID, &r.Payload, &r.Metadata) + if err != nil { + return sql.Row{}, errors.Wrap(err, "could not scan message row") + } + + msg := message.NewMessage(string(r.UUID), r.Payload) + + if r.Metadata != nil { + err = json.Unmarshal(r.Metadata, &msg.Metadata) + if err != nil { + return sql.Row{}, errors.Wrap(err, "could not unmarshal metadata as JSON") + } + } + + r.Msg = msg + r.ExtraData = map[string]any{ + "transaction_id": transactionID, + } + + return r, nil +} + +func (s PostgreSQLSchema) MessagesTable(topic string) string { + if s.GenerateMessagesTableName != nil { + return s.GenerateMessagesTableName(topic) + } + return fmt.Sprintf(`"watermill_%s"`, topic) +} + +func (s PostgreSQLSchema) SubscribeIsolationLevel() stdSQL.IsolationLevel { + // For Postgres Repeatable Read is enough. + return stdSQL.LevelSerializable +} + +func PostgresDBConnection(cfg config.Sql) (*stdSQL.DB, error) { + dbURL := "postgres://" + cfg.Username + ":" + cfg.Password + "@" + cfg.Host + ":" + strconv.Itoa(cfg.Port) + "/" + cfg.Db + "?" + cfg.QueryString + + db, err := stdSQL.Open("postgres", dbURL) + if err != nil { + return nil, err + } + return db, err + +} diff --git a/go.mod b/go.mod index cded92d..fbf3194 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/Improwised/golang-api -go 1.19 +go 1.21 + +toolchain go1.21.6 require ( clevergo.tech/jsend v1.1.3 @@ -45,6 +47,7 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.5 // indirect github.com/Rican7/retry v0.3.1 // indirect + github.com/ThreeDotsLabs/watermill-sql/v2 v2.0.0 github.com/andybalholm/brotli v1.0.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v3 v3.2.2 // indirect @@ -92,7 +95,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -100,7 +103,6 @@ require ( github.com/rabbitmq/amqp091-go v1.9.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.4.4 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sony/gobreaker v0.5.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 9976686..0e85c46 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGB cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/kms v1.15.5 h1:pj1sRfut2eRbD9pFRjNnPNg/CzJPuQAzUujMIM1vVeM= +cloud.google.com/go/kms v1.15.5/go.mod h1:cU2H5jnp6G2TDpUGZyqTCoy1n16fbubHZjmVXSMtwDI= cloud.google.com/go/pubsub v1.36.0 h1:cgaJ0mgwEM0YNNATXFXSnLfji6XcMieTc2mRjH1ZYdY= cloud.google.com/go/pubsub v1.36.0/go.mod h1:qQvGW4ANjuYcOpTMTy5+u6HBIoJF00cPfQ/ubMcc/D8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -34,6 +36,8 @@ github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0 h1:/KYEjLlLx6nW3jn6AEcwAlWkPW github.com/ThreeDotsLabs/watermill-kafka/v2 v2.5.0/go.mod h1:w+9jhI7x5ZP67ceSUIIpkgLzjAakotfHX4sWyqsKVjs= github.com/ThreeDotsLabs/watermill-redisstream v1.2.2 h1:/fFHagJiObMBbYIDrygRoAq+RxqLPcQZdGi6b0ViG08= github.com/ThreeDotsLabs/watermill-redisstream v1.2.2/go.mod h1:ZRe0VpA0Ho/4MESUrXdqJMaWtiWhi4emxIYpqsxi98Y= +github.com/ThreeDotsLabs/watermill-sql/v2 v2.0.0 h1:wswlLYY0Jc0tloj3lty4Y+VTEA8AM1vYfrIDwWtqyJk= +github.com/ThreeDotsLabs/watermill-sql/v2 v2.0.0/go.mod h1:83l/4sKaLHwoHJlrAsDLaXcHN+QOHHntAAyabNmiuO4= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef h1:2JGTg6JapxP9/R33ZaagQtAM4EkkSYnIAlOG5EI8gkM= @@ -53,6 +57,8 @@ github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= +github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -78,6 +84,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.2 h1:QkIBuU5k+x7/QXPvPPnWXWlCdaBFApVqftFV6k087DA= +github.com/envoyproxy/protoc-gen-validate v1.0.2/go.mod h1:GpiZQP3dDbg4JouG/NNS7QWXpgx6x8QiMKdmN72jogE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= @@ -90,7 +98,6 @@ github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3Bop github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs= github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -124,7 +131,6 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -148,14 +154,12 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= @@ -175,6 +179,22 @@ github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/C github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= +github.com/jackc/chunkreader/v2 v2.0.1/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= +github.com/jackc/pgconn v1.6.4 h1:S7T6cx5o2OqmxdHaXLH1ZeD1SbI8jBznyYE9Ec0RCQ8= +github.com/jackc/pgconn v1.6.4/go.mod h1:w2pne1C2tZgP+TvjqLpOigGzNqjBgQW9dUw/4Chex78= +github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= +github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgproto3/v2 v2.0.2 h1:q1Hsy66zh4vuNsajBUF2PNqfAMMfxU5mk594lPE9vjY= +github.com/jackc/pgproto3/v2 v2.0.2/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgtype v1.4.2 h1:t+6LWm5eWPLX1H5Se702JSBcirq6uWa4jiG4wV1rAWY= +github.com/jackc/pgtype v1.4.2/go.mod h1:JCULISAZBFGrHaOXIIFiyfzW5VY0GRitRr8NeJsrdig= +github.com/jackc/pgx/v4 v4.8.1 h1:SUbCLP2pXvf/Sr/25KsuI4aTxiFYIvpfk4l6aTSdyCw= +github.com/jackc/pgx/v4 v4.8.1/go.mod h1:4HOLxrl8wToZJReD04/yB20GDwf4KBYETvlHciCnwW0= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -329,6 +349,8 @@ github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT github.com/xdg-go/scram v1.1.0/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.einride.tech/aip v0.65.0 h1:aqKEV1g9diXcR6DAxBVZoJn6ho8SuC+TOZFXzuu7kLU= +go.einride.tech/aip v0.65.0/go.mod h1:wcRZ57XFEvERWLPy9VqDBtXc/ZFj7ugsd32F5o8Th+s= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama v0.31.0 h1:J8jI81RCB7U9a3qsTZXM/38XrvbLJCye6J32bfQctYY= @@ -337,13 +359,13 @@ go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.4 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= -go.opentelemetry.io/otel v1.6.1 h1:6r1YrcTenBvYa1x491d0GGpTVBsNECmrc/K6b+zDeis= go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ= go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/trace v1.6.1 h1:f8c93l5tboBYZna1nWk0W9DYyMzJXDWdZcJZ0Kb400U= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0= go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= @@ -379,7 +401,6 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -430,7 +451,6 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -452,13 +472,12 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.157.0 h1:ORAeqmbrrozeyw5NjnMxh7peHO0UzV4wWYSwZeCUb20= google.golang.org/api v0.157.0/go.mod h1:+z4v4ufbZ1WEpld6yMGHyggs+PmAHiaLNj5ytP3N01g= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -488,8 +507,6 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/watermill/publisher.go b/pkg/watermill/publisher.go index ca51466..04c3b45 100644 --- a/pkg/watermill/publisher.go +++ b/pkg/watermill/publisher.go @@ -8,9 +8,11 @@ import ( "github.com/Improwised/golang-api/cli/workers" "github.com/Improwised/golang-api/config" + "github.com/Improwised/golang-api/database" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amqp/v2/pkg/amqp" "github.com/ThreeDotsLabs/watermill-googlecloud/pkg/googlecloud" + "github.com/ThreeDotsLabs/watermill-sql/v2/pkg/sql" "github.com/ThreeDotsLabs/watermill-kafka/v2/pkg/kafka" "github.com/ThreeDotsLabs/watermill-redisstream/pkg/redisstream" @@ -18,12 +20,15 @@ import ( "github.com/redis/go-redis/v9" ) -type WatermillPubliser struct { +type WatermillPublisher struct { publisher message.Publisher } - -func InitPubliser(cfg config.AppConfig) (*WatermillPubliser, error) { +// ? single database or mualtipal database +func InitPublisher(cfg config.AppConfig,isDeadLetterQ bool) (*WatermillPublisher, error) { logger = watermill.NewStdLogger(cfg.MQ.Debug, cfg.MQ.Track) + if isDeadLetterQ { + return initSqlPub(cfg) + } switch cfg.MQ.Dialect { case "amqp": return initAmqpPub(cfg) @@ -36,8 +41,11 @@ func InitPubliser(cfg config.AppConfig) (*WatermillPubliser, error) { case "googlecloud": return initGoogleCloudPub(cfg) + + case "sql": + return initSqlPub(cfg) default: - return &WatermillPubliser{}, nil + return &WatermillPublisher{}, nil } } @@ -45,7 +53,7 @@ func InitPubliser(cfg config.AppConfig) (*WatermillPubliser, error) { // send message into queue using topic name // // struct must from worker package(/cli/workers) -func (wp *WatermillPubliser) Publish(topic string, inputStruct interface{}) error { +func (wp *WatermillPublisher) Publish(topic string, inputStruct interface{}) error { // if broker is not set then return nil if wp.publisher == nil { return nil @@ -69,14 +77,14 @@ func (wp *WatermillPubliser) Publish(topic string, inputStruct interface{}) erro return err } -func initAmqpPub(cfg config.AppConfig) (*WatermillPubliser, error) { +func initAmqpPub(cfg config.AppConfig) (*WatermillPublisher, error) { amqpConfig := amqp.NewDurableQueueConfig(cfg.MQ.Amqp.AmqbUrl) publisher, err := amqp.NewPublisher(amqpConfig, logger) - return &WatermillPubliser{publisher: publisher}, err + return &WatermillPublisher{publisher: publisher}, err } -func initRedisPub(cfg config.AppConfig) (*WatermillPubliser, error) { +func initRedisPub(cfg config.AppConfig) (*WatermillPublisher, error) { pubClient := redis.NewClient(&redis.Options{ Addr: cfg.MQ.Redis.RedisUrl, Username: cfg.MQ.Redis.UserName, @@ -89,10 +97,10 @@ func initRedisPub(cfg config.AppConfig) (*WatermillPubliser, error) { }, logger, ) - return &WatermillPubliser{publisher: publisher}, err + return &WatermillPublisher{publisher: publisher}, err } -func initKafkaPub(cfg config.AppConfig) (*WatermillPubliser, error) { +func initKafkaPub(cfg config.AppConfig) (*WatermillPublisher, error) { publisher, err := kafka.NewPublisher( kafka.PublisherConfig{ Brokers: cfg.MQ.Kafka.KafkaBroker, @@ -101,16 +109,70 @@ func initKafkaPub(cfg config.AppConfig) (*WatermillPubliser, error) { }, logger, ) - return &WatermillPubliser{publisher: publisher}, err + return &WatermillPublisher{publisher: publisher}, err } -func initGoogleCloudPub(cfg config.AppConfig) (*WatermillPubliser, error) { +func initGoogleCloudPub(cfg config.AppConfig) (*WatermillPublisher, error) { publisher, err := googlecloud.NewPublisher(googlecloud.PublisherConfig{ ProjectID: cfg.MQ.GoogleCloud.ProjectID, ConnectTimeout: 10 * time.Second, PublishTimeout: 10 * time.Second, - Marshaler: googlecloud.DefaultMarshalerUnmarshaler{}, + Marshaler: googlecloud.DefaultMarshalerUnmarshaler{}, }, logger) - return &WatermillPubliser{publisher: publisher}, err + return &WatermillPublisher{publisher: publisher}, err +} + +func initSqlPub(cfg config.AppConfig) (*WatermillPublisher, error) { + switch cfg.MQ.Sql.Dialect { + case "postgres": + return initPostgresPub(cfg) + case "mysql": + return initMysqlPub(cfg) + default: + return nil, nil + } + +} + +func initPostgresPub(cfg config.AppConfig) (*WatermillPublisher, error) { + db, err := database.PostgresDBConnection(cfg.MQ.Sql) + if err != nil { + return nil, err + } + publisher, err := sql.NewPublisher( + db, + sql.PublisherConfig{ + // we are customizing schema adapter because default schema adapter has some issue + SchemaAdapter: database.PostgreSQLSchema{}, + AutoInitializeSchema: true, + }, + + logger, + ) + if err != nil { + return nil, err + } + return &WatermillPublisher{publisher: publisher}, nil +} + +func initMysqlPub(cfg config.AppConfig) (*WatermillPublisher, error) { + db, err := database.MysqlDBConnection(cfg.MQ.Sql) + if err != nil { + return nil, err + } + publisher, err := sql.NewPublisher( + db, + sql.PublisherConfig{ + SchemaAdapter: database.MySQLSchema{}, + AutoInitializeSchema: true, + }, + + logger, + ) + if err != nil { + return nil, err + } + + return &WatermillPublisher{publisher: publisher}, nil } diff --git a/pkg/watermill/subscriber.go b/pkg/watermill/subscriber.go index f347217..0210137 100644 --- a/pkg/watermill/subscriber.go +++ b/pkg/watermill/subscriber.go @@ -7,10 +7,12 @@ import ( "cloud.google.com/go/pubsub" "github.com/Improwised/golang-api/config" + "github.com/Improwised/golang-api/database" "github.com/Shopify/sarama" "github.com/ThreeDotsLabs/watermill" "github.com/ThreeDotsLabs/watermill-amqp/pkg/amqp" + "github.com/ThreeDotsLabs/watermill-sql/v2/pkg/sql" "github.com/ThreeDotsLabs/watermill-googlecloud/pkg/googlecloud" "github.com/ThreeDotsLabs/watermill-kafka/v2/pkg/kafka" @@ -28,8 +30,11 @@ type WatermillSubscriber struct { Router *message.Router } -func InitSubscriber(cfg config.AppConfig) (*WatermillSubscriber, error) { +func InitSubscriber(cfg config.AppConfig, isDeadLetterQ bool) (*WatermillSubscriber, error) { logger = watermill.NewStdLogger(cfg.MQ.Debug, cfg.MQ.Track) + if isDeadLetterQ { + return initSqlSub(cfg) + } switch cfg.MQ.Dialect { case "amqp": return initAmqpSub(cfg) @@ -43,6 +48,8 @@ func InitSubscriber(cfg config.AppConfig) (*WatermillSubscriber, error) { case "googlecloud": return initGoogleCloudSub(cfg) + case "sql": + return initSqlSub(cfg) default: return nil, nil } @@ -55,12 +62,12 @@ func (ws *WatermillSubscriber) InitRouter(cfg config.AppConfig, delayTime, MaxRe return nil, err } - pub, err := InitPubliser(cfg) + pub, err := InitPublisher(cfg, true) if err != nil { return nil, err } - poq, err := middleware.PoisonQueue(pub.publisher, cfg.MQ.DeadQueue) + poq, err := middleware.PoisonQueue(pub.publisher, cfg.MQ.DeadLetterQ) if err != nil { return nil, err } @@ -83,7 +90,7 @@ func (ws *WatermillSubscriber) InitRouter(cfg config.AppConfig, delayTime, MaxRe return ws, nil } -func (ws *WatermillSubscriber) Run(topic string, handlerFunc message.NoPublishHandlerFunc) error { +func (ws *WatermillSubscriber) Run(topic,handlerName string ,handlerFunc message.NoPublishHandlerFunc) error { if ws.Subscriber == nil { return fmt.Errorf("subscriber is nil") } @@ -97,7 +104,7 @@ func (ws *WatermillSubscriber) Run(topic string, handlerFunc message.NoPublishHa } ws.Router.AddNoPublisherHandler( - "handler", + handlerName, topic, ws.Subscriber, handlerFunc, @@ -179,3 +186,57 @@ func initGoogleCloudSub(cfg config.AppConfig) (*WatermillSubscriber, error) { return &WatermillSubscriber{Subscriber: subscriber}, err } + +func initSqlSub(cfg config.AppConfig) (*WatermillSubscriber, error) { + switch cfg.MQ.Sql.Dialect { + case "postgres": + return initPostgresSub(cfg) + + case "mysql": + return initMysqlSub(cfg) + + default: + return nil, nil + } +} + +func initPostgresSub(cfg config.AppConfig) (*WatermillSubscriber, error) { + db, err := database.PostgresDBConnection(cfg.MQ.Sql) + if err != nil { + return nil, err + } + subscriber, err := sql.NewSubscriber( + db, + sql.SubscriberConfig{ + SchemaAdapter: database.PostgreSQLSchema{}, + OffsetsAdapter: sql.DefaultPostgreSQLOffsetsAdapter{}, + InitializeSchema: true, + }, + logger, + ) + if err != nil { + return nil, err + } + return &WatermillSubscriber{Subscriber: subscriber}, err +} + +func initMysqlSub(cfg config.AppConfig) (*WatermillSubscriber, error) { + db, err := database.MysqlDBConnection(cfg.MQ.Sql) + if err != nil { + return nil, err + } + subscriber, err := sql.NewSubscriber( + db, + sql.SubscriberConfig{ + SchemaAdapter: database.MySQLSchema{}, + OffsetsAdapter: sql.DefaultMySQLOffsetsAdapter{}, + InitializeSchema: true, + }, + logger, + ) + if err != nil { + return nil, err + } + + return &WatermillSubscriber{Subscriber: subscriber}, err +} diff --git a/routes/main.go b/routes/main.go index 3d2152e..a6a8c63 100644 --- a/routes/main.go +++ b/routes/main.go @@ -20,7 +20,7 @@ import ( var mu sync.Mutex // Setup func -func Setup(app *fiber.App, goqu *goqu.Database, logger *zap.Logger, config config.AppConfig, events *events.Events, pMetrics *pMetrics.PrometheusMetrics, pub *watermill.WatermillPubliser) error { +func Setup(app *fiber.App, goqu *goqu.Database, logger *zap.Logger, config config.AppConfig, events *events.Events, pMetrics *pMetrics.PrometheusMetrics, pub *watermill.WatermillPublisher) error { mu.Lock() app.Use(middlewares.LogHandler(logger, pMetrics)) @@ -69,7 +69,7 @@ func setupAuthController(v1 fiber.Router, goqu *goqu.Database, logger *zap.Logge return nil } -func setupUserController(v1 fiber.Router, goqu *goqu.Database, logger *zap.Logger, middlewares middlewares.Middleware, events *events.Events, pub *watermill.WatermillPubliser) error { +func setupUserController(v1 fiber.Router, goqu *goqu.Database, logger *zap.Logger, middlewares middlewares.Middleware, events *events.Events, pub *watermill.WatermillPublisher) error { userController, err := controller.NewUserController(goqu, logger, events, pub) if err != nil { return err From 5d9e082dcc5b05b7de68f2e54e048541bdd792ff Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Wed, 31 Jan 2024 11:46:45 +0530 Subject: [PATCH 14/16] fix: data type in mysql schema --- database/watermill_mysql_schema.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/watermill_mysql_schema.go b/database/watermill_mysql_schema.go index 703bf98..720b6f8 100644 --- a/database/watermill_mysql_schema.go +++ b/database/watermill_mysql_schema.go @@ -26,7 +26,7 @@ func (s MySQLSchema) SchemaInitializingQueries(topic string) []string { "`offset` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,", "`uuid` VARCHAR(36) NOT NULL,", "`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,", - "`payload` BINARY(255) NULL,", + "`payload` BLOB DEFAULT NULL,", "`metadata` JSON DEFAULT NULL", ");", }, "\n") From e39978a2b35e6316119417a8edf5b595301f59f5 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Wed, 31 Jan 2024 12:24:32 +0530 Subject: [PATCH 15/16] fix: message payload set when job fail --- cli/worker.go | 1 + cli/workers/worker_handler.go | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/cli/worker.go b/cli/worker.go index 00f41ae..9b59cab 100644 --- a/cli/worker.go +++ b/cli/worker.go @@ -47,6 +47,7 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command } // run worker with topic(queue name) and process function + workers := &workers.QConfig{Topic: topic,MaxRetries: retryCount,} err = router.Run(topic, cfg.MQ.HandlerName, workers.Process) return err diff --git a/cli/workers/worker_handler.go b/cli/workers/worker_handler.go index ab430aa..feee5d2 100644 --- a/cli/workers/worker_handler.go +++ b/cli/workers/worker_handler.go @@ -4,10 +4,16 @@ import ( "bytes" "encoding/gob" "encoding/json" + "strconv" "github.com/ThreeDotsLabs/watermill/message" ) +type QConfig struct { + Topic string + MaxRetries int +} + func init() { for _, v := range RegisterWorkerStruct() { gob.Register(v) @@ -28,7 +34,7 @@ type Handler interface { } // process all worker struct and call Handle function according to struct -func Process(msg *message.Message) error { +func (q QConfig) Process(msg *message.Message) error { buf := bytes.NewBuffer(msg.Payload) dec := gob.NewDecoder(buf) @@ -37,14 +43,32 @@ func Process(msg *message.Message) error { if err != nil { return err } + // Store the JSON payload in the msg so that it can be persisted in the database in case the job fails. - msg.Payload, err = json.Marshal(result) - if err != nil { - return err + CurrentRetryCount := GetSetRetryCount(msg) + if CurrentRetryCount == q.MaxRetries { + msg.Payload, err = json.Marshal(result) + if err != nil { + return err + } } + if err := result.Handle(); err != nil { return err } msg.Ack() return nil } + +// set/get retry count from message metadata +func GetSetRetryCount(msg *message.Message) int { + count := msg.Metadata.Get("max_retry_count") + if count == "" { + msg.Metadata.Set("max_retry_count", "0") + count = "0" + return 0 + } + countInt, _ := strconv.Atoi(count) + msg.Metadata.Set("max_retry_count", strconv.Itoa(countInt+1)) + return countInt + 1 +} From 8f2a6af57a2c26f354bdd680631ac8ebdc858018 Mon Sep 17 00:00:00 2001 From: chintansakhiya Date: Thu, 1 Feb 2024 19:00:28 +0530 Subject: [PATCH 16/16] fix: change type from interface to Handler in publish func, change datatype of mysql schema, remove unnecessary logic --- README.md | 2 +- cli/dead_letter_queue.go | 28 +++------------------------ cli/worker.go | 1 - cli/workers/worker_handler.go | 31 +----------------------------- database/watermill_mysql_schema.go | 2 +- pkg/watermill/publisher.go | 14 +++----------- 6 files changed, 9 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index f1fe7c7..2af914a 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Migrations are like **version control for your database**, allowing your team to - #### Multiple Message Queue Broker Support - - We are supporting 5 types of message queue broker at this time `rabbitmq`, `redis`, `googleCloud`,`sql(postges,mysql)` & `kafka` + - We are supporting 5 types of message queue broker at this time `rabbitmq`, `redis`, `googleCloud`,`sql(postgres,mysql)` & `kafka` - It allows us to switch to message queue broker without changing too much stuff. - Watermill package allows us to do that. - We have environment variable `MQ_DIALECT` where you can set to message queue broker type. diff --git a/cli/dead_letter_queue.go b/cli/dead_letter_queue.go index 3c163a9..99d83d7 100644 --- a/cli/dead_letter_queue.go +++ b/cli/dead_letter_queue.go @@ -1,14 +1,11 @@ package cli import ( - "encoding/json" - "log" - "go.uber.org/zap" + "github.com/Improwised/golang-api/cli/workers" "github.com/Improwised/golang-api/config" "github.com/Improwised/golang-api/pkg/watermill" - "github.com/ThreeDotsLabs/watermill/message" "github.com/spf13/cobra" ) @@ -35,29 +32,10 @@ func GetDeadQueueCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Comm } // run worker with topic(queue name) and process function - err = subscriber.Run(cfg.MQ.DeadLetterQ,"dead_letter_queue" ,HandleFailJob) + // it will run failed job until it get success + err = subscriber.Run(cfg.MQ.DeadLetterQ, "dead_letter_queue", workers.Process) return err }, } return workerCommand } - -func HandleFailJob(msg *message.Message) error { - // get fail job details from metadata - var result DeadLetterQ - metadata := make(map[string]string) - for k, v := range msg.Metadata { - metadata[k] = v - } - jsonString, err := json.Marshal(metadata) - if err != nil { - return err - } - err = json.Unmarshal(jsonString, &result) - if err != nil { - return err - } - - log.Printf("Failed job handler: %s, reason: %s, subscriber: %s, topic: %s", result.Handler, result.Reason, result.Subscriber, result.Topic) - return nil -} diff --git a/cli/worker.go b/cli/worker.go index 9b59cab..00f41ae 100644 --- a/cli/worker.go +++ b/cli/worker.go @@ -47,7 +47,6 @@ func GetWorkerCommandDef(cfg config.AppConfig, logger *zap.Logger) cobra.Command } // run worker with topic(queue name) and process function - workers := &workers.QConfig{Topic: topic,MaxRetries: retryCount,} err = router.Run(topic, cfg.MQ.HandlerName, workers.Process) return err diff --git a/cli/workers/worker_handler.go b/cli/workers/worker_handler.go index feee5d2..1507896 100644 --- a/cli/workers/worker_handler.go +++ b/cli/workers/worker_handler.go @@ -3,17 +3,10 @@ package workers import ( "bytes" "encoding/gob" - "encoding/json" - "strconv" "github.com/ThreeDotsLabs/watermill/message" ) -type QConfig struct { - Topic string - MaxRetries int -} - func init() { for _, v := range RegisterWorkerStruct() { gob.Register(v) @@ -34,7 +27,7 @@ type Handler interface { } // process all worker struct and call Handle function according to struct -func (q QConfig) Process(msg *message.Message) error { +func Process(msg *message.Message) error { buf := bytes.NewBuffer(msg.Payload) dec := gob.NewDecoder(buf) @@ -44,31 +37,9 @@ func (q QConfig) Process(msg *message.Message) error { return err } - // Store the JSON payload in the msg so that it can be persisted in the database in case the job fails. - CurrentRetryCount := GetSetRetryCount(msg) - if CurrentRetryCount == q.MaxRetries { - msg.Payload, err = json.Marshal(result) - if err != nil { - return err - } - } - if err := result.Handle(); err != nil { return err } msg.Ack() return nil } - -// set/get retry count from message metadata -func GetSetRetryCount(msg *message.Message) int { - count := msg.Metadata.Get("max_retry_count") - if count == "" { - msg.Metadata.Set("max_retry_count", "0") - count = "0" - return 0 - } - countInt, _ := strconv.Atoi(count) - msg.Metadata.Set("max_retry_count", strconv.Itoa(countInt+1)) - return countInt + 1 -} diff --git a/database/watermill_mysql_schema.go b/database/watermill_mysql_schema.go index 720b6f8..5bd4447 100644 --- a/database/watermill_mysql_schema.go +++ b/database/watermill_mysql_schema.go @@ -26,7 +26,7 @@ func (s MySQLSchema) SchemaInitializingQueries(topic string) []string { "`offset` BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY,", "`uuid` VARCHAR(36) NOT NULL,", "`created_at` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,", - "`payload` BLOB DEFAULT NULL,", + "`payload` LONGBLOB DEFAULT NULL,", "`metadata` JSON DEFAULT NULL", ");", }, "\n") diff --git a/pkg/watermill/publisher.go b/pkg/watermill/publisher.go index 04c3b45..82a2461 100644 --- a/pkg/watermill/publisher.go +++ b/pkg/watermill/publisher.go @@ -3,7 +3,6 @@ package watermill import ( "bytes" "encoding/gob" - "fmt" "time" "github.com/Improwised/golang-api/cli/workers" @@ -23,8 +22,8 @@ import ( type WatermillPublisher struct { publisher message.Publisher } -// ? single database or mualtipal database -func InitPublisher(cfg config.AppConfig,isDeadLetterQ bool) (*WatermillPublisher, error) { + +func InitPublisher(cfg config.AppConfig, isDeadLetterQ bool) (*WatermillPublisher, error) { logger = watermill.NewStdLogger(cfg.MQ.Debug, cfg.MQ.Track) if isDeadLetterQ { return initSqlPub(cfg) @@ -53,19 +52,13 @@ func InitPublisher(cfg config.AppConfig,isDeadLetterQ bool) (*WatermillPublisher // send message into queue using topic name // // struct must from worker package(/cli/workers) -func (wp *WatermillPublisher) Publish(topic string, inputStruct interface{}) error { +func (wp *WatermillPublisher) Publish(topic string, handle workers.Handler) error { // if broker is not set then return nil if wp.publisher == nil { return nil } var network bytes.Buffer enc := gob.NewEncoder(&network) - var handle workers.Handler - - handle, ok := inputStruct.(workers.Handler) - if !ok { - return fmt.Errorf("struct is not of type workers.Handler") - } err := enc.Encode(&handle) if err != nil { @@ -83,7 +76,6 @@ func initAmqpPub(cfg config.AppConfig) (*WatermillPublisher, error) { return &WatermillPublisher{publisher: publisher}, err } - func initRedisPub(cfg config.AppConfig) (*WatermillPublisher, error) { pubClient := redis.NewClient(&redis.Options{ Addr: cfg.MQ.Redis.RedisUrl,