From 4b355efc8d7c1d9a0178a78d632cf9595cf06a72 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Mon, 8 Mar 2021 14:36:07 -0800 Subject: [PATCH 1/3] Recover from broken RDS connections. --- cmd/keymasterd/storage.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/keymasterd/storage.go b/cmd/keymasterd/storage.go index f521d5e4..e02abc2d 100644 --- a/cmd/keymasterd/storage.go +++ b/cmd/keymasterd/storage.go @@ -129,6 +129,8 @@ func initDBPostgres(state *RuntimeState) (err error) { return err } } + // Ensure that broken connections are replaced. + state.db.SetConnMaxLifetime(state.Config.ProfileStorage.SyncInterval >> 1) return nil } From fdcdac68c8fd0377006ca94f190b85233018b627 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Tue, 9 Mar 2021 06:06:48 -0800 Subject: [PATCH 2/3] Clarify storage logging messages. --- cmd/keymasterd/storage.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/keymasterd/storage.go b/cmd/keymasterd/storage.go index e02abc2d..e5c51d67 100644 --- a/cmd/keymasterd/storage.go +++ b/cmd/keymasterd/storage.go @@ -384,7 +384,7 @@ func (state *RuntimeState) GetUsers() ([]string, bool, error) { } return dbMessage.Names, false, dbMessage.Err case <-time.After(state.remoteDBQueryTimeout): - logger.Printf("GOT a timeout") + logger.Println("GetUsers: timed out on primary DB") stmtText := getUsersStmt["sqlite"] stmt, err := state.cacheDB.Prepare(stmtText) if err != nil { @@ -397,7 +397,7 @@ func (state *RuntimeState) GetUsers() ([]string, bool, error) { if dbErr != nil { logger.Printf("Problem with db = '%s'", err) } else { - logger.Println("GOT data from db cache") + logger.Println("GetUsers: got data from DB cache") } return names, true, dbErr } @@ -465,7 +465,7 @@ func (state *RuntimeState) LoadUserProfile(username string) ( metricLogExternalServiceDuration("storage-read", time.Since(start)) profileBytes = dbMessage.ProfileBytes case <-time.After(state.remoteDBQueryTimeout): - logger.Printf("GOT a timeout") + logger.Println("LoadUserProfile: timed out on primary DB") fromCache = true // load from cache stmtText := loadUserProfileStmt["sqlite"] @@ -486,7 +486,7 @@ func (state *RuntimeState) LoadUserProfile(username string) ( return nil, false, true, err } } - logger.Printf("GOT data from db cache") + logger.Println("LoadUserProfile: got data from DB cache") } logger.Debugf(10, "profile bytes len=%d", len(profileBytes)) //gobReader := bytes.NewReader(fileBytes) @@ -645,7 +645,7 @@ func (state *RuntimeState) GetSigned(username string, } jwsData = dbMessage.JWSData case <-time.After(state.remoteDBQueryTimeout): - logger.Printf("GOT a timeout") + logger.Println("GetSigned: timed out on primary DB") // load from cache stmtText := getSignedUserDataStmt["sqlite"] stmt, err := state.cacheDB.Prepare(stmtText) @@ -666,9 +666,9 @@ func (state *RuntimeState) GetSigned(username string, return false, "", err } } - logger.Printf("GOT data from db cache") + logger.Println("GetSigned: got data from DB cache") } - logger.Printf("GOT some jwsdata data") + logger.Println("GetSigned: got jwsdata") storageJWT, err := state.getStorageDataFromStorageStringDataJWT(jwsData) if err != nil { logger.Debugf(2, "failed to get storage data %s data=%s", err, jwsData) From e5106dd07e8053a7eac2b65ad6ea42ae9357d694 Mon Sep 17 00:00:00 2001 From: Richard Gooch Date: Tue, 9 Mar 2021 19:25:15 -0800 Subject: [PATCH 3/3] Make DB connection max lifetime configurable. --- cmd/keymasterd/config.go | 1 + cmd/keymasterd/storage.go | 13 ++++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cmd/keymasterd/config.go b/cmd/keymasterd/config.go index 1a10a3fe..c33e7701 100644 --- a/cmd/keymasterd/config.go +++ b/cmd/keymasterd/config.go @@ -144,6 +144,7 @@ type OpenIDConnectIDPConfig struct { type ProfileStorageConfig struct { AwsSecretId string `yaml:"aws_secret_id"` + ConnectionLifetime time.Duration `yaml:"connection_lifetime"` StorageUrl string `yaml:"storage_url"` SyncDelay time.Duration `yaml:"sync_delay"` SyncInterval time.Duration `yaml:"sync_interval"` diff --git a/cmd/keymasterd/storage.go b/cmd/keymasterd/storage.go index e5c51d67..b36c4028 100644 --- a/cmd/keymasterd/storage.go +++ b/cmd/keymasterd/storage.go @@ -23,6 +23,9 @@ const ( profileDBFilename = "userProfiles.sqlite3" cachedDBFilename = "cachedDB.sqlite3" + dbConnectionLifetimeDefault = time.Minute * 15 + dbConnectionLifetimeMaximum = time.Hour + dbSyncDelayDefault = time.Second * 3 dbSyncDelayMinimum = time.Second dbSyncDelayMaximum = time.Minute @@ -47,6 +50,14 @@ func (config *ProfileStorageConfig) setSyncLimits() { } else if config.SyncInterval > dbSyncIntervalMaximum { config.SyncInterval = dbSyncIntervalMaximum } + if config.ConnectionLifetime < 1 { + config.ConnectionLifetime = dbConnectionLifetimeDefault + } + if config.ConnectionLifetime < config.SyncInterval { + config.ConnectionLifetime = config.SyncInterval + } else if config.ConnectionLifetime > dbConnectionLifetimeMaximum { + config.ConnectionLifetime = dbConnectionLifetimeMaximum + } } func (state *RuntimeState) expandStorageUrl() error { @@ -130,7 +141,7 @@ func initDBPostgres(state *RuntimeState) (err error) { } } // Ensure that broken connections are replaced. - state.db.SetConnMaxLifetime(state.Config.ProfileStorage.SyncInterval >> 1) + state.db.SetConnMaxLifetime(state.Config.ProfileStorage.ConnectionLifetime) return nil }