From e81ee39c85f17d87fd1a3d7d4db64327b1dd21a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Thu, 10 Oct 2024 12:19:00 +0000 Subject: [PATCH 1/4] fix(script-log)!: Changing config fils structure to prep for future features Partly implements #1265 --- src/config/config-gen-api-docs.yaml | 11 +++++++--- src/config/production_template.yaml | 11 +++++++--- src/lib/assert/config-file-schema.js | 32 +++++++++++++++++++++++----- 3 files changed, 43 insertions(+), 11 deletions(-) diff --git a/src/config/config-gen-api-docs.yaml b/src/config/config-gen-api-docs.yaml index 856d8f8e..34e80042 100644 --- a/src/config/config-gen-api-docs.yaml +++ b/src/config/config-gen-api-docs.yaml @@ -130,9 +130,14 @@ Butler: # NOTE: Use an absolute path when running Butler as a standalone executable! scriptLog: storeOnDisk: - reloadTaskFailure: - enable: false - logDirectory: ... + clientManaged: + reloadTaskFailure: + enable: false + logDirectory: /path/to/scriptlogs/qseow + qsCloud: + apPReloadFailure: + enable: false + logDirectory: /path/to/scriptlogs/qscloud # Qlik Sense related links used in notification messages qlikSenseUrls: diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index 4df8d66f..01de4b03 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -147,9 +147,14 @@ Butler: # NOTE: Use an absolute path when running Butler as a standalone executable! scriptLog: storeOnDisk: - reloadTaskFailure: - enable: false - logDirectory: /path/to/scriptlogs + clientManaged: + reloadTaskFailure: + enable: false + logDirectory: /path/to/scriptlogs/qseow + qsCloud: + apPReloadFailure: + enable: false + logDirectory: /path/to/scriptlogs/qscloud # Qlik Sense related links used in notification messages qlikSenseUrls: diff --git a/src/lib/assert/config-file-schema.js b/src/lib/assert/config-file-schema.js index fb1744a8..c0e5ac4c 100755 --- a/src/lib/assert/config-file-schema.js +++ b/src/lib/assert/config-file-schema.js @@ -347,17 +347,39 @@ export const confifgFileSchema = { storeOnDisk: { type: 'object', properties: { - reloadTaskFailure: { + clientManaged: { type: 'object', properties: { - enable: { type: 'boolean' }, - logDirectory: { type: 'string' }, + reloadTaskFailure: { + type: 'object', + properties: { + enable: { type: 'boolean' }, + logDirectory: { type: 'string' }, + }, + required: ['enable', 'logDirectory'], + additionalProperties: false, + }, }, - required: ['enable', 'logDirectory'], + required: ['reloadTaskFailure'], additionalProperties: false, }, + qsCloud: { + type: 'object', + properties: { + appReloadFailure: { + type: 'object', + properties: { + enable: { type: 'boolean' }, + logDirectory: { type: 'string' }, + }, + required: ['enable', 'logDirectory'], + additionalProperties: false, + }, + }, + required: ['appReloadFailure'], + }, }, - required: ['reloadTaskFailure'], + required: ['clientManaged', 'qsCloud'], additionalProperties: false, }, }, From 8e64041fbd15302275362b28e62a866245b68503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Thu, 10 Oct 2024 15:51:39 +0000 Subject: [PATCH 2/4] Updated telemetry payload with missing properties --- src/lib/telemetry.js | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/lib/telemetry.js b/src/lib/telemetry.js index ae34d343..f1025d33 100644 --- a/src/lib/telemetry.js +++ b/src/lib/telemetry.js @@ -41,6 +41,10 @@ const callRemoteURL = async () => { let api_slackPostMessage = 'null'; let influxDb_reloadTaskFailure = 'null'; + let influxDb_reloadTaskSuccess = 'null'; + + let scriptLog_qseow_reloadTaskFailure = 'null'; + let scriptLog_qscloud_appReloadFailure = 'null'; let teamsNotification_reloadTaskFailure = 'null'; let teamsNotification_reloadTaskAborted = 'null'; @@ -203,6 +207,18 @@ const callRemoteURL = async () => { influxDb_reloadTaskFailure = globals.config.get('Butler.influxDb.reloadTaskFailure.enable'); } + if (globals.config.has('Butler.influxDb.reloadTaskSuccess.enable')) { + influxDb_reloadTaskSuccess = globals.config.get('Butler.influxDb.reloadTaskSuccess.enable'); + } + + if (globals.config.has('Butler.scriptLog.storeOnDisk.clientManaged.reloadTaskFailure.enable')) { + scriptLog_qseow_reloadTaskFailure = globals.config.get('Butler.scriptLog.storeOnDisk.clientManaged.reloadTaskFailure.enable'); + } + + if (globals.config.has('Butler.scriptLog.storeOnDisk.qsCloud.appReloadFailure.enable')) { + scriptLog_qscloud_appReloadFailure = globals.config.get('Butler.scriptLog.storeOnDisk.qsCloud.appReloadFailure.enable'); + } + if (globals.config.has('Butler.teamsNotification.reloadTaskFailure.enable')) { teamsNotification_reloadTaskFailure = globals.config.get('Butler.teamsNotification.reloadTaskFailure.enable'); } @@ -373,6 +389,10 @@ const callRemoteURL = async () => { feature_apiSlackPostMessage: api_slackPostMessage, feature_influxDbReloadTaskFailure: influxDb_reloadTaskFailure, + feature_influxDbReloadTaskSuccess: influxDb_reloadTaskSuccess, + + feature_scriptLogQseowReloadTaskFailure: scriptLog_qseow_reloadTaskFailure, + feature_scriptLogQsCloudAppReloadFailure: scriptLog_qscloud_appReloadFailure, feature_teamsNotificationReloadTaskFailure: teamsNotification_reloadTaskFailure, feature_teamsNotificationReloadTaskAborted: teamsNotification_reloadTaskAborted, @@ -431,6 +451,16 @@ const callRemoteURL = async () => { : {}, influxDbReloadTaskFailure: influxDb_reloadTaskFailure, + influxDbReloadTaskSuccess: influxDb_reloadTaskSuccess, + + scriptLogStoreOnDisk: { + qseow: { + reloadTaskFailure: scriptLog_qseow_reloadTaskFailure, + }, + qsCloud: { + appReloadFailure: scriptLog_qscloud_appReloadFailure, + }, + }, teamsNotificationReloadTaskFailure: teamsNotification_reloadTaskFailure, teamsNotificationReloadTaskAborted: teamsNotification_reloadTaskAborted, From 206a7137f6f868bdff1580e4a749f80055d36abb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Thu, 10 Oct 2024 16:01:04 +0000 Subject: [PATCH 3/4] Remove typo and reference to Butler SOS --- src/config/production_template.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index 01de4b03..28d6ee41 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -152,7 +152,7 @@ Butler: enable: false logDirectory: /path/to/scriptlogs/qseow qsCloud: - apPReloadFailure: + appReloadFailure: enable: false logDirectory: /path/to/scriptlogs/qscloud @@ -980,14 +980,14 @@ Butler: qlikSenseCloud: # Settings for Qlik Sense Cloud integration enable: false event: - mqtt: # Which QS Cloud tenant should Butler SOS receive events from, in the form of MQTT messages? + mqtt: # Which QS Cloud tenant should Butler receive events from, in the form of MQTT messages? tenant: id: tenant.region.qlikcloud.com tenantUrl: https://tenant.region.qlikcloud.com authType: jwt # Authentication type used to connect to the tenant. Valid options are "jwt" auth: jwt: - token: # JWT token used to authenticate Butler SOS when connecting to the tenant + token: # JWT token used to authenticate Butler when connecting to the tenant # Qlik Sense Cloud related links used in notification messages qlikSenseUrls: qmc: @@ -1081,7 +1081,7 @@ Butler: - name: X-Qlik-User # Header used to identify what user connection to QRS is made as value: UserDirectory=Internal;UserId=sa_repository # What user connection to QRS is made as rejectUnauthorized: false # Set to false to ignore warnings/errors caused by Qlik Sense's self-signed certificates. - # Set to true if the Qlik Sense root CA is available on the computer where Butler SOS is running. + # Set to true if the Qlik Sense root CA is available on the computer where Butler is running. configQRS: authentication: certificates @@ -1093,7 +1093,7 @@ Butler: - name: X-Qlik-User # Header used to identify what user connection to QRS is made as value: UserDirectory=Internal;UserId=sa_repository # What user connection to QRS is made as rejectUnauthorized: false # Set to false to ignore warnings/errors caused by Qlik Sense's self-signed certificates. - # Set to true if the Qlik Sense root CA is available on the computer where Butler SOS is running. + # Set to true if the Qlik Sense root CA is available on the computer where Butler is running. configDirectories: qvdPath: From 30e6a37558e495b92ddcd04a921f96c07599a9bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Mon, 14 Oct 2024 18:48:03 +0000 Subject: [PATCH 4/4] Fix storing script logs on disk for QSEoW and Cloud --- src/config/config-gen-api-docs.yaml | 2 +- src/config/production_template.yaml | 12 +- src/lib/qscloud/api/appreloadinfo.js | 50 +- src/lib/qscloud/email_notification_qscloud.js | 567 +++++++++--------- .../qscloud/mqtt_event_app_reload_finished.js | 469 ++++++--------- src/lib/qscloud/slack_notification_qscloud.js | 2 +- src/lib/qseow/scriptlog.js | 2 +- src/udp/udp_handlers.js | 2 +- 8 files changed, 484 insertions(+), 622 deletions(-) diff --git a/src/config/config-gen-api-docs.yaml b/src/config/config-gen-api-docs.yaml index 34e80042..837c87cf 100644 --- a/src/config/config-gen-api-docs.yaml +++ b/src/config/config-gen-api-docs.yaml @@ -135,7 +135,7 @@ Butler: enable: false logDirectory: /path/to/scriptlogs/qseow qsCloud: - apPReloadFailure: + appReloadFailure: enable: false logDirectory: /path/to/scriptlogs/qscloud diff --git a/src/config/production_template.yaml b/src/config/production_template.yaml index 28d6ee41..d139466b 100644 --- a/src/config/production_template.yaml +++ b/src/config/production_template.yaml @@ -156,12 +156,13 @@ Butler: enable: false logDirectory: /path/to/scriptlogs/qscloud - # Qlik Sense related links used in notification messages + # Qlik Sense (client-managed) related links used in notification messages qlikSenseUrls: qmc: hub: appBaseUrl: //sense/app # Base URL for Qlik Sense apps, for example http://sense.mycompany.net/sense/app. App ID will be appended to this URL. + # Links available as template variables in notification messages genericUrls: - id: ptarmiganlabs_com linkText: Ptarmigan Labs home page @@ -370,7 +371,7 @@ Butler: # These alerts will be sent irrespective of the alertEnableByCustomProperty.enable setting. alertEnabledByEmailAddress: customPropertyName: 'Butler_SuccessAlertSendToEmail' - rateLimit: 60 # Min seconds between emails for a given taskID. Defaults to 5 minutes. + rateLimit: 60 # Min seconds between emails for a given taskID/recipient combo. Defaults to 5 minutes. headScriptLogLines: 15 tailScriptLogLines: 25 priority: high # high/normal/low @@ -1003,10 +1004,9 @@ Butler: tag: Butler - Send Teams alert if app reload fails basicContentOnly: false webhookURL: - messageType: formatted # formatted / basic basicMsgTemplate: 'Qlik Sense Cloud app reload failed: "{{appName}}"' # Only needed if message type = basic - rateLimit: 15 # Min seconds between emails for a given taskID. Defaults to 5 minutes. + rateLimit: 15 # Min seconds between emails for a given appId. Defaults to 5 minutes. headScriptLogLines: 15 tailScriptLogLines: 15 templateFile: /path/to/teams_templates/failed-reload-qscloud-workflow.handlebars @@ -1023,7 +1023,7 @@ Butler: channel: sense-task-failure # Slack channel to which task failure notifications are sent messageType: formatted # formatted / basic. Formatted means that template file below will be used to create the message. basicMsgTemplate: 'Qlik Sense Cloud app reload failed: "{{appName}}"' # Only needed if message type = basic - rateLimit: 60 # Min seconds between emails for a given taskID. Defaults to 5 minutes. + rateLimit: 60 # Min seconds between emails for a given appId. Defaults to 5 minutes. headScriptLogLines: 10 tailScriptLogLines: 20 templateFile: /path/to/slack_templates/failed-reload-qscloud.handlebars @@ -1049,7 +1049,7 @@ Butler: excludeOwner: user: # - email: daniel@somecompany.com - rateLimit: 60 # Min seconds between emails for a given taskID. Defaults to 5 minutes. + rateLimit: 60 # Min seconds between emails for a given appId/recipient combo. Defaults to 5 minutes. headScriptLogLines: 15 tailScriptLogLines: 25 priority: high # high/normal/low diff --git a/src/lib/qscloud/api/appreloadinfo.js b/src/lib/qscloud/api/appreloadinfo.js index 1109ef26..7f268d43 100644 --- a/src/lib/qscloud/api/appreloadinfo.js +++ b/src/lib/qscloud/api/appreloadinfo.js @@ -7,7 +7,7 @@ import globals from '../../../globals.js'; // Parameters: // - appId: Qlik Sense Cloud app ID // - reloadId: Qlik Sense Cloud reload ID -export async function getQlikSenseCloudAppReloadScriptLog(appId, reloadId, headLineCount, tailLineCount) { +export async function getQlikSenseCloudAppReloadScriptLog(appId, reloadId) { try { // Set up Qlik Sense Cloud API configuration const axiosConfig = { @@ -26,29 +26,11 @@ export async function getQlikSenseCloudAppReloadScriptLog(appId, reloadId, headL // Get number of lines in scriptLogFull const scriptLogLineCount = scriptLogFull.length; - let scriptLogHead = ''; - let scriptLogTail = ''; - - if (headLineCount > 0) { - scriptLogHead = scriptLogFull.slice(0, headLineCount).join('\r\n'); - } - - if (tailLineCount > 0) { - scriptLogTail = scriptLogFull.slice(Math.max(scriptLogFull.length - tailLineCount, 0)).join('\r\n'); - } - - globals.logger.debug(`QLIK SENSE CLOUD GET SCRIPT LOG: Script log head:\n${scriptLogHead}`); - globals.logger.debug(`QLIK SENSE CLOUD GET SCRIPT LOG: Script log tails:\n${scriptLogTail}`); - globals.logger.verbose('QLIK SENSE CLOUD GET SCRIPT LOG: Done getting script log'); return { scriptLogFull, scriptLogSize: scriptLogLineCount, - scriptLogHead, - scriptLogHeadCount: headLineCount, - scriptLogTail, - scriptLogTailCount: tailLineCount, }; } catch (err) { globals.logger.error(`QLIK SENSE CLOUD GET SCRIPT LOG: ${err}`); @@ -56,6 +38,36 @@ export async function getQlikSenseCloudAppReloadScriptLog(appId, reloadId, headL } } +// Function to get script log head lines +// Parameters: +// - scriptLogFull: Full script log as array +// - headLineCount: Number of lines to get from head +export function getQlikSenseCloudAppReloadScriptLogHead(scriptLogFull, headLineCount) { + if (headLineCount > 0) { + const scriptLogHead = scriptLogFull.slice(0, headLineCount).join('\r\n'); + globals.logger.debug(`QLIK SENSE CLOUD GET SCRIPT LOG: Script log head:\n${scriptLogHead}`); + + return scriptLogHead; + } else { + return ''; + } +} + +// Function to get script log tail lines +// Parameters: +// - scriptLogFull: Full script log as array +// - tailLineCount: Number of lines to get from tail +export function getQlikSenseCloudAppReloadScriptLogTail(scriptLogFull, tailLineCount) { + if (tailLineCount > 0) { + const scriptLogTail = scriptLogFull.slice(Math.max(scriptLogFull.length - tailLineCount, 0)).join('\r\n'); + globals.logger.debug(`QLIK SENSE CLOUD GET SCRIPT LOG: Script log tails:\n${scriptLogTail}`); + + return scriptLogTail; + } else { + return ''; + } +} + // Function to get general info/status/result for a specific Qlik Sense Cloud reload // Parameters: // - reloadId: Qlik Sense Cloud reload ID diff --git a/src/lib/qscloud/email_notification_qscloud.js b/src/lib/qscloud/email_notification_qscloud.js index 4bc9da7f..da7dac4f 100644 --- a/src/lib/qscloud/email_notification_qscloud.js +++ b/src/lib/qscloud/email_notification_qscloud.js @@ -74,320 +74,305 @@ function getAppReloadFailedEmailConfig() { } // Function to send Qlik Sense Cloud app reload failed alert as email -export function sendQlikSenseCloudAppReloadFailureNotificationEmail(reloadParams) { - rateLimiterMemoryFailedReloads - .consume(reloadParams.reloadId, 1) - .then(async (rateLimiterRes) => { - try { - globals.logger.info( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting check passed for failed task notification. App name: "${reloadParams.appName}"`, - ); - globals.logger.verbose( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"`, - ); +export async function sendQlikSenseCloudAppReloadFailureNotificationEmail(reloadParams) { + try { + globals.logger.info( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting check passed for failed task notification. App name: "${reloadParams.appName}"`, + ); + globals.logger.verbose( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"`, + ); - // Logic for determining if alert email should be sent or not - // 1. If config setting Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.enabled is false, do not send email - // 2. If config setting Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.enable is true, - // ...only send email if the app has the tag specified in Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.tag - // - // Logic for determining list of email recipients - // 1. Should alert emails be sent for all failed reload tasks? - // Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.enable = true => only send email if app has tag - // Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.enable = false => send email for all failed reloads - // 1. Yes: Add system-wide list of recipients to send list. This list is defined in Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.recipients[] - // 2. No: Does the app whose reload failed have a tag that enables alert emails? - // 1. Yes: Add system-wide list of recipients to send list. This list is defined in Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.recipients[] - // 2. No: Don't add recpients to send list - // 2. Should app owners get alerts? Determined by Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.enable, which is a boolean - // 1. Yes: Should *all* app owners get alerts? Determined by Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.includeOwner.includeAll, which is a boolean - // 1. Yes: Add app owner's email address to app owner send list - // 2. No: Is the app owner included in the Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.includeOwner.user[] array? - // 1. Yes: Add app owner's email address to app owner send list - // 2. No: Don't add app owner's email address to app owner send list - // 2. Is there an app owner exclusion list? Determined by Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.excludeOwner.[] - // 1. Yes: Is the app owner's email address in the exclusion list? - // 1. Yes: Remove the app owner's email address from the app owner send list (if it's there) - // 3. Add app owner send list to main send list - // 4. Remove any duplicate email addresses from the main send list - - // Make sure email sending is enabled in the config file and that we have all required settings - emailConfig = getAppReloadFailedEmailConfig(); - if (emailConfig === false) { - return 1; - } + // Logic for determining if alert email should be sent or not + // 1. If config setting Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.enabled is false, do not send email + // 2. If config setting Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.enable is true, + // ...only send email if the app has the tag specified in Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.tag + // + // Logic for determining list of email recipients + // 1. Should alert emails be sent for all failed reload tasks? + // Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.enable = true => only send email if app has tag + // Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.alertEnableByTag.enable = false => send email for all failed reloads + // 1. Yes: Add system-wide list of recipients to send list. This list is defined in Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.recipients[] + // 2. No: Does the app whose reload failed have a tag that enables alert emails? + // 1. Yes: Add system-wide list of recipients to send list. This list is defined in Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.recipients[] + // 2. No: Don't add recpients to send list + // 2. Should app owners get alerts? Determined by Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.enable, which is a boolean + // 1. Yes: Should *all* app owners get alerts? Determined by Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.includeOwner.includeAll, which is a boolean + // 1. Yes: Add app owner's email address to app owner send list + // 2. No: Is the app owner included in the Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.includeOwner.user[] array? + // 1. Yes: Add app owner's email address to app owner send list + // 2. No: Don't add app owner's email address to app owner send list + // 2. Is there an app owner exclusion list? Determined by Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.appOwnerAlert.excludeOwner.[] + // 1. Yes: Is the app owner's email address in the exclusion list? + // 1. Yes: Remove the app owner's email address from the app owner send list (if it's there) + // 3. Add app owner send list to main send list + // 4. Remove any duplicate email addresses from the main send list + + // Make sure email sending is enabled in the config file and that we have all required settings + emailConfig = getAppReloadFailedEmailConfig(); + if (emailConfig === false) { + return 1; + } - // Get send list based on logic described above - let globalSendList = []; + // Get send list based on logic described above + let globalSendList = []; - // Get recipients based on app tags (or for all failed reloads) - if (emailConfig.emailAlertByTagEnable === false) { - // Email alerts are enabled for all failed app reloads, not just those with a specific tag set - if (emailConfig?.globalSendList?.length < 0) { - // Add global send list from YAML config file to main send list - globalSendList.push(...emailConfig.globalSendList); - } - } else { - // Check if app has the tag that enables email alerts. If not found, do not add anything to the main send list - // The app tag names are in reloadParams.meta.tags[].name - const alertTag = emailConfig.emailAlertByTagName; - const appTags = reloadParams.appItems.meta.tags; - const appHasAlertTag = appTags.find((tag) => tag.name === alertTag); - - if (appTags === undefined || appTags?.length === 0 || appHasAlertTag === undefined) { - globals.logger.warn( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: App [${reloadParams.appId}] "${reloadParams.appName}" does not have the tag "${alertTag}" set. Not sending alert email based on app tag.`, - ); - } else if (appHasAlertTag !== undefined) { - if (emailConfig?.globalSendList?.length > 0) { - // Add global send list from YAML config file to main send list - globalSendList.push(...emailConfig.globalSendList); - } - } + // Get recipients based on app tags (or for all failed reloads) + if (emailConfig.emailAlertByTagEnable === false) { + // Email alerts are enabled for all failed app reloads, not just those with a specific tag set + if (emailConfig?.globalSendList?.length < 0) { + // Add global send list from YAML config file to main send list + globalSendList.push(...emailConfig.globalSendList); + } + } else { + // Check if app has the tag that enables email alerts. If not found, do not add anything to the main send list + // The app tag names are in reloadParams.meta.tags[].name + const alertTag = emailConfig.emailAlertByTagName; + const appTags = reloadParams.appItems.meta.tags; + const appHasAlertTag = appTags.find((tag) => tag.name === alertTag); + + if (appTags === undefined || appTags?.length === 0 || appHasAlertTag === undefined) { + globals.logger.warn( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: App [${reloadParams.appId}] "${reloadParams.appName}" does not have the tag "${alertTag}" set. Not sending alert email based on app tag.`, + ); + } else if (appHasAlertTag !== undefined) { + if (emailConfig?.globalSendList?.length > 0) { + // Add global send list from YAML config file to main send list + globalSendList.push(...emailConfig.globalSendList); } + } + } - // Get app owner info - const appOwner = await getQlikSenseCloudUserInfo(reloadParams.ownerId); + // Get app owner info + const appOwner = await getQlikSenseCloudUserInfo(reloadParams.ownerId); - // Get recipients based on app owner settings - // Build separate list of app owner email addresses - if (emailConfig.appOwnerAlert.enable === true) { - let appOwnerSendList = []; + // Get recipients based on app owner settings + // Build separate list of app owner email addresses + if (emailConfig.appOwnerAlert.enable === true) { + let appOwnerSendList = []; - if (appOwner.email === undefined || appOwner?.email?.length === 0) { - globals.logger.warn( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: App owner email address is not set for app [${reloadParams.appId}] "${reloadParams.appName}". Not sending alert email to app owner "${appOwner.name}".`, - ); - } else { - // App owner email address exists. + if (appOwner.email === undefined || appOwner?.email?.length === 0) { + globals.logger.warn( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: App owner email address is not set for app [${reloadParams.appId}] "${reloadParams.appName}". Not sending alert email to app owner "${appOwner.name}".`, + ); + } else { + // App owner email address exists. + + // Should *all* app owners get alerts? + if (emailConfig.appOwnerAlert.includeOwner.includeAll === true) { + // Add app owner's email address to app owner send list. + // Only do this if the email address length is greater than 0 + appOwnerSendList.push(appOwner.email); + } else { + // Check if app owner's email address is in the include list + const appOwnerIncludeList = emailConfig.appOwnerAlert.includeOwner?.user; + if (appOwnerIncludeList !== undefined && appOwnerIncludeList?.length > 0) { + const appOwnerIsIncluded = appOwnerIncludeList.find((owner) => owner.email === appOwner.email); - // Should *all* app owners get alerts? - if (emailConfig.appOwnerAlert.includeOwner.includeAll === true) { - // Add app owner's email address to app owner send list. - // Only do this if the email address length is greater than 0 + if (appOwnerIsIncluded !== undefined) { + // Add app owner's email address to app owner send list appOwnerSendList.push(appOwner.email); - } else { - // Check if app owner's email address is in the include list - const appOwnerIncludeList = emailConfig.appOwnerAlert.includeOwner?.user; - if (appOwnerIncludeList !== undefined && appOwnerIncludeList?.length > 0) { - const appOwnerIsIncluded = appOwnerIncludeList.find((owner) => owner.email === appOwner.email); - - if (appOwnerIsIncluded !== undefined) { - // Add app owner's email address to app owner send list - appOwnerSendList.push(appOwner.email); - } - } } + } + } - // Now evaluate the exclusion list - const appOwnerExcludeList = emailConfig.appOwnerAlert.excludeOwner?.user; - if (appOwnerExcludeList !== undefined && appOwnerExcludeList?.length > 0) { - // Exclusion list found. - // Remove all entries in exclude list from app owner send list - appOwnerSendList = appOwnerSendList.filter((ownerEmail) => { - const appOwnerIsExcluded = appOwnerExcludeList.find((exclude) => exclude.email === ownerEmail); - if (appOwnerIsExcluded === undefined) { - return ownerEmail; - } - }); + // Now evaluate the exclusion list + const appOwnerExcludeList = emailConfig.appOwnerAlert.excludeOwner?.user; + if (appOwnerExcludeList !== undefined && appOwnerExcludeList?.length > 0) { + // Exclusion list found. + // Remove all entries in exclude list from app owner send list + appOwnerSendList = appOwnerSendList.filter((ownerEmail) => { + const appOwnerIsExcluded = appOwnerExcludeList.find((exclude) => exclude.email === ownerEmail); + if (appOwnerIsExcluded === undefined) { + return ownerEmail; } - - // Add app owner send list to main send list - globalSendList.push(...appOwnerSendList); - } + }); } - // Remove any duplicate email addresses from the main send list - globalSendList = [...new Set(globalSendList)]; + // Add app owner send list to main send list + globalSendList.push(...appOwnerSendList); + } + } - // Check if we have any email addresses to send to - if (globalSendList?.length === 0) { - globals.logger.warn( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: No email addresses found to send alert email for app [${reloadParams.appId}] "${reloadParams.appName}".`, - ); - return false; - } + // Remove any duplicate email addresses from the main send list + globalSendList = [...new Set(globalSendList)]; - if (isSmtpConfigOk() === false) { - return false; - } + // Check if we have any email addresses to send to + if (globalSendList?.length === 0) { + globals.logger.warn( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: No email addresses found to send alert email for app [${reloadParams.appId}] "${reloadParams.appName}".`, + ); + return false; + } - // Get script logs, if enabled in the config file - // If the value is false, the script log could not be obtained - let scriptLogData = {}; - - if (reloadParams.scriptLog === false) { - scriptLogData = { - scriptLogFull: [], - scriptLogSize: 0, - scriptLogHead: '', - scriptLogHeadCount: 0, - scriptLogTail: '', - scriptLogTailCount: 0, - }; - } else { - // Reduce script log lines to only the ones we want to send to email - scriptLogData.scriptLogHeadCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.headScriptLogLines', - ); - scriptLogData.scriptLogTailCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.tailScriptLogLines', - ); + if (isSmtpConfigOk() === false) { + return false; + } - if (reloadParams.scriptLog?.scriptLogFull?.length > 0) { - // Get length of script log (character count) - scriptLogData.scriptLogSize = reloadParams.scriptLog.scriptLogFull.length; - - // Get the first and last n lines of the script log - scriptLogData.scriptLogHead = reloadParams.scriptLog.scriptLogFull - .slice(0, reloadParams.scriptLog.scriptLogHeadCount) - .join('\r\n'); - - scriptLogData.scriptLogTail = reloadParams.scriptLog.scriptLogFull - .slice(Math.max(reloadParams.scriptLog.scriptLogFull.length - reloadParams.scriptLog.scriptLogTailCount, 0)) - .join('\r\n'); - } else { - scriptLogData.scriptLogHead = ''; - scriptLogData.scriptLogTail = ''; - scriptLogData.scriptLogSize = 0; - } + // Get script logs, if enabled in the config file + // If the value is false, the script log could not be obtained + let scriptLogData = {}; + + if (reloadParams.scriptLog === false) { + scriptLogData = { + scriptLogFull: [], + scriptLogSize: 0, + scriptLogHead: '', + scriptLogHeadCount: 0, + scriptLogTail: '', + scriptLogTailCount: 0, + }; + } else { + // Reduce script log lines to only the ones we want to send to email + scriptLogData.scriptLogHeadCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.headScriptLogLines', + ); + scriptLogData.scriptLogTailCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.tailScriptLogLines', + ); - globals.logger.debug( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`, - ); - } + if (reloadParams.scriptLog?.scriptLogFull?.length > 0) { + // Get length of script log (character count) + scriptLogData.scriptLogSize = reloadParams.scriptLog.scriptLogFull.length; + + // Get the first and last n lines of the script log + scriptLogData.scriptLogHead = reloadParams.scriptLog.scriptLogFull + .slice(0, reloadParams.scriptLog.scriptLogHeadCount) + .join('\r\n'); + + scriptLogData.scriptLogTail = reloadParams.scriptLog.scriptLogFull + .slice(Math.max(reloadParams.scriptLog.scriptLogFull.length - reloadParams.scriptLog.scriptLogTailCount, 0)) + .join('\r\n'); + } else { + scriptLogData.scriptLogHead = ''; + scriptLogData.scriptLogTail = ''; + scriptLogData.scriptLogSize = 0; + } - // Format log message line breaks to work in HTML email - if (reloadParams.reloadInfo.log !== undefined) { - // Replace \n with \r\n - reloadParams.reloadInfo.log = reloadParams.reloadInfo.log.replace(/(\r\n|\n|\r)/gm, '\r\n'); - } + globals.logger.debug(`EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Script log data:\n${JSON.stringify(scriptLogData, null, 2)}`); + } - // Get Sense URLs from config file. Can be used as template fields. - const senseUrls = getQlikSenseCloudUrls(); + // Format log message line breaks to work in HTML email + if (reloadParams.reloadInfo.log !== undefined) { + // Replace \n with \r\n + reloadParams.reloadInfo.log = reloadParams.reloadInfo.log.replace(/(\r\n|\n|\r)/gm, '\r\n'); + } - // Get generic URLs from config file. Can be used as template fields. - let genericUrls = globals.config.get('Butler.genericUrls'); - if (!genericUrls) { - // No URLs defined in the config file. Set to empty array - genericUrls = []; - } + // Get Sense URLs from config file. Can be used as template fields. + const senseUrls = getQlikSenseCloudUrls(); - // These are the template fields that can be used in email body - const templateContext = { - tenantId: reloadParams.tenantId, - tenantComment: reloadParams.tenantComment, - tenantUrl: reloadParams.tenantUrl, - - userId: reloadParams.userId, - userName: appOwner === undefined ? 'Unknown' : appOwner.name, - - appId: reloadParams.appId, - appName: reloadParams.appName, - appDescription: reloadParams.appInfo.attributes.description, - appUrl: reloadParams.appUrl, - appHasSectionAccess: reloadParams.appInfo.attributes.hasSectionAccess, - appIsPublished: reloadParams.appInfo.attributes.published, - appPublishTime: reloadParams.appInfo.attributes.publishTime, - appThumbnail: reloadParams.appInfo.attributes.thumbnail, - - reloadTrigger: reloadParams.reloadTrigger, - source: reloadParams.source, - eventType: reloadParams.eventType, - eventTypeVersion: reloadParams.eventTypeVersion, - endedWithMemoryConstraint: reloadParams.endedWithMemoryConstraint, - isDirectQueryMode: reloadParams.isDirectQueryMode, - isPartialReload: reloadParams.isPartialReload, - isSessionApp: reloadParams.isSessionApp, - isSkipStore: reloadParams.isSkipStore, - - peakMemoryBytes: reloadParams.peakMemoryBytes.toLocaleString(), - reloadId: reloadParams.reloadId, - rowLimit: reloadParams.rowLimit.toLocaleString(), - statements: reloadParams.statements, - status: reloadParams.status, - usageDuration: reloadParams.duration, - sizeMemoryBytes: reloadParams.sizeMemory.toLocaleString(), - appFileSize: reloadParams.appItems.resourceSize.appFile.toLocaleString(), - - errorCode: reloadParams.reloadInfo.errorCode, - errorMessage: reloadParams.reloadInfo.errorMessage, - logMessage: reloadParams.reloadInfo.log, - executionDuration: reloadParams.reloadInfo.executionDuration, - executionStartTime: reloadParams.reloadInfo.executionStartTime, - executionStopTime: reloadParams.reloadInfo.executionStopTime, - executionStatusText: reloadParams.reloadInfo.status, - scriptLogSize: scriptLogData.scriptLogSize.toLocaleString(), - scriptLogHead: scriptLogData.scriptLogHead, - scriptLogTail: scriptLogData.scriptLogTail, - scriptLogTailCount: scriptLogData.scriptLogTailCount, - scriptLogHeadCount: scriptLogData.scriptLogHeadCount, - - qlikSenseQMC: senseUrls.qmcUrl, - qlikSenseHub: senseUrls.hubUrl, - genericUrls, - - appOwnerName: appOwner.name, - appOwnerUserId: appOwner.id, - appOwnerPicture: appOwner.picture, - appOwnerEmail: appOwner.email, - }; - - // Send alert emails - // Take into account rate limiting, basing it on appId + email address - for (const recipientEmailAddress of globalSendList) { - rateLimiterMemoryFailedReloads - .consume(`${reloadParams.taskId}|${recipientEmailAddress}`, 1) - // eslint-disable-next-line no-loop-func - .then(async (rateLimiterRes) => { - try { - globals.logger.info( - `EMAIL ALERT - QS CLOUD: Rate limiting check passed for failed app reload notification. App name: "${reloadParams.appName}", email: "${recipientEmailAddress}"`, - ); - globals.logger.debug( - `EMAIL ALERT - QS CLOUD: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"`, - ); - - // Only send email if there is an actual email address - if (recipientEmailAddress?.length > 0) { - sendEmail( - emailConfig.fromAddress, - [recipientEmailAddress], - emailConfig.priority, - emailConfig.subject, - emailConfig.bodyFileDirectory, - emailConfig.htmlTemplateFile, - templateContext, - ); - } else { - globals.logger.warn( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: No email address found for app [${reloadParams.appId}] "${reloadParams.appName}". Not sending alert email.`, - ); - } - } catch (err) { - globals.logger.error(`EMAIL ALERT - QS CLOUD APP RELOAD FAILED: ${err}`); - } - }) - .catch((err) => { - globals.logger.warn( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting failed. Not sending reload notification email for app [${reloadParams.appId}] "${reloadParams.appName}"`, + // Get generic URLs from config file. Can be used as template fields. + let genericUrls = globals.config.get('Butler.genericUrls'); + if (!genericUrls) { + // No URLs defined in the config file. Set to empty array + genericUrls = []; + } + + // These are the template fields that can be used in email body + const templateContext = { + tenantId: reloadParams.tenantId, + tenantComment: reloadParams.tenantComment, + tenantUrl: reloadParams.tenantUrl, + + userId: reloadParams.userId, + userName: appOwner === undefined ? 'Unknown' : appOwner.name, + + appId: reloadParams.appId, + appName: reloadParams.appName, + appDescription: reloadParams.appInfo.attributes.description, + appUrl: reloadParams.appUrl, + appHasSectionAccess: reloadParams.appInfo.attributes.hasSectionAccess, + appIsPublished: reloadParams.appInfo.attributes.published, + appPublishTime: reloadParams.appInfo.attributes.publishTime, + appThumbnail: reloadParams.appInfo.attributes.thumbnail, + + reloadTrigger: reloadParams.reloadTrigger, + source: reloadParams.source, + eventType: reloadParams.eventType, + eventTypeVersion: reloadParams.eventTypeVersion, + endedWithMemoryConstraint: reloadParams.endedWithMemoryConstraint, + isDirectQueryMode: reloadParams.isDirectQueryMode, + isPartialReload: reloadParams.isPartialReload, + isSessionApp: reloadParams.isSessionApp, + isSkipStore: reloadParams.isSkipStore, + + peakMemoryBytes: reloadParams.peakMemoryBytes.toLocaleString(), + reloadId: reloadParams.reloadId, + rowLimit: reloadParams.rowLimit.toLocaleString(), + statements: reloadParams.statements, + status: reloadParams.status, + usageDuration: reloadParams.duration, + sizeMemoryBytes: reloadParams.sizeMemory.toLocaleString(), + appFileSize: reloadParams.appItems.resourceSize.appFile.toLocaleString(), + + errorCode: reloadParams.reloadInfo.errorCode, + errorMessage: reloadParams.reloadInfo.errorMessage, + logMessage: reloadParams.reloadInfo.log, + executionDuration: reloadParams.reloadInfo.executionDuration, + executionStartTime: reloadParams.reloadInfo.executionStartTime, + executionStopTime: reloadParams.reloadInfo.executionStopTime, + executionStatusText: reloadParams.reloadInfo.status, + scriptLogSize: scriptLogData.scriptLogSize.toLocaleString(), + scriptLogHead: scriptLogData.scriptLogHead, + scriptLogTail: scriptLogData.scriptLogTail, + scriptLogTailCount: scriptLogData.scriptLogTailCount, + scriptLogHeadCount: scriptLogData.scriptLogHeadCount, + + qlikSenseQMC: senseUrls.qmcUrl, + qlikSenseHub: senseUrls.hubUrl, + genericUrls, + + appOwnerName: appOwner.name, + appOwnerUserId: appOwner.id, + appOwnerPicture: appOwner.picture, + appOwnerEmail: appOwner.email, + }; + + // Send alert emails + // Take into account rate limiting, basing it on appId + email address + for (const recipientEmailAddress of globalSendList) { + rateLimiterMemoryFailedReloads + .consume(`${reloadParams.appId}|${recipientEmailAddress}`, 1) + // eslint-disable-next-line no-loop-func + .then(async (rateLimiterRes) => { + try { + globals.logger.info( + `EMAIL ALERT - QS CLOUD: Rate limiting check passed for failed app reload notification. App name: "${reloadParams.appName}", email: "${recipientEmailAddress}"`, + ); + globals.logger.debug(`EMAIL ALERT - QS CLOUD: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"`); + + // Only send email if there is an actual email address + if (recipientEmailAddress?.length > 0) { + sendEmail( + emailConfig.fromAddress, + [recipientEmailAddress], + emailConfig.priority, + emailConfig.subject, + emailConfig.bodyFileDirectory, + emailConfig.htmlTemplateFile, + templateContext, ); - globals.logger.debug( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting details "${JSON.stringify(err, null, 2)}"`, + } else { + globals.logger.warn( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: No email address found for app [${reloadParams.appId}] "${reloadParams.appName}". Not sending alert email.`, ); - }); - } - } catch (err) { - globals.logger.error(`EMAIL ALERT - QS CLOUD APP RELOAD FAILED: ${err}`); - } - return true; - }) - .catch((rateLimiterRes) => { - globals.logger.warn( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting failed. Not sending reload notification email for app [${reloadParams.appId}] "${reloadParams.appName}"`, - ); - globals.logger.debug( - `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting details "${JSON.stringify(rateLimiterRes, null, 2)}"`, - ); - }); + } + } catch (err) { + globals.logger.error(`EMAIL ALERT - QS CLOUD APP RELOAD FAILED: ${err}`); + } + }) + .catch((err) => { + globals.logger.warn( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting failed. Not sending reload notification email for app [${reloadParams.appId}] "${reloadParams.appName}"`, + ); + globals.logger.debug( + `EMAIL ALERT - QS CLOUD APP RELOAD FAILED: Rate limiting details "${JSON.stringify(err, null, 2)}"`, + ); + }); + } + } catch (err) { + globals.logger.error(`EMAIL ALERT - QS CLOUD APP RELOAD FAILED: ${err}`); + } + + return true; } diff --git a/src/lib/qscloud/mqtt_event_app_reload_finished.js b/src/lib/qscloud/mqtt_event_app_reload_finished.js index 5aa51cc4..06060f65 100644 --- a/src/lib/qscloud/mqtt_event_app_reload_finished.js +++ b/src/lib/qscloud/mqtt_event_app_reload_finished.js @@ -1,5 +1,13 @@ +import path from 'path'; +import fs from 'fs'; + import globals from '../../globals.js'; -import { getQlikSenseCloudAppReloadScriptLog, getQlikSenseCloudAppReloadInfo } from './api/appreloadinfo.js'; +import { + getQlikSenseCloudAppReloadScriptLog, + getQlikSenseCloudAppReloadInfo, + getQlikSenseCloudAppReloadScriptLogHead, + getQlikSenseCloudAppReloadScriptLogTail, +} from './api/appreloadinfo.js'; import { getQlikSenseCloudAppInfo, getQlikSenseCloudAppMetadata, getQlikSenseCloudAppItems } from './api/app.js'; import { sendQlikSenseCloudAppReloadFailureNotificationTeams } from './msteams_notification_qscloud.js'; import { sendQlikSenseCloudAppReloadFailureNotificationSlack } from './slack_notification_qscloud.js'; @@ -23,7 +31,7 @@ export async function handleQlikSenseCloudAppReloadFinished(message) { const appId = message.extensions.topLevelResourceId; // Get info about the app reload from the message sent by Qlik Sense Cloud - const { source, eventType, eventTypeVersion } = message; + const { source, eventType, eventTime, eventTypeVersion } = message; const { ownerId, tenantId, userId } = message.extensions; const { duration, @@ -74,11 +82,133 @@ export async function handleQlikSenseCloudAppReloadFinished(message) { let appMetadata = {}; let appItems = {}; - // App reload did fail. Send enabled notifications/alerts + // App reload did fail. Send enabled notifications/alerts, store script log etc logger.info(`QLIK SENSE CLOUD: App reload failed. App ID=[${appId}] name="${message.data.name}"`); // Are notifications from QS Cloud enabled? if (globals.config.get('Butler.qlikSenseCloud.enable') === true) { + // Get info that will be needed later from QS CLoud APIs + // This includes: + // - Reload script log + // - Reload info + // - App info + + // Script log is available via "GET /v1/apps/{appId}/reloads/logs/{reloadId}" + // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-reloads-logs-reloadId + try { + scriptLog = await getQlikSenseCloudAppReloadScriptLog(appId, reloadId); + } catch (err) { + logger.error(`QLIK SENSE CLOUD: Could not get app reload script log. Error=${JSON.stringify(err, null, 2)}`); + } + + // If return value is false, the script log could not be obtained + if (scriptLog === false) { + logger.warn(`QLIK SENSE CLOUD: Could not get app reload script log. App ID="${appId}", reload ID="${reloadId}"`); + } else { + logger.verbose(`QLIK SENSE CLOUD: App reload script log obtained. App ID="${appId}", reload ID="${reloadId}"`); + } + logger.debug(`QLIK SENSE CLOUD: App reload script log: ${scriptLog}`); + + // Reload info is available via "GET /v1/reloads/{reloadId}" + // https://qlik.dev/apis/rest/reloads/#get-v1-reloads-reloadId + try { + reloadInfo = await getQlikSenseCloudAppReloadInfo(reloadId); + reloadTrigger = reloadInfo.type; + + logger.verbose(`QLIK SENSE CLOUD: App reload info obtained. App ID="${appId}", reload ID="${reloadId}"`); + logger.debug(`QLIK SENSE CLOUD: App reload info: ${JSON.stringify(reloadInfo, null, 2)}`); + } catch (err) { + logger.error(`QLIK SENSE CLOUD: Could not get app reload info. Error=${JSON.stringify(err, null, 2)}`); + } + + // App info is available via "GET /v1/apps/{appId}" + // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId + try { + appInfo = await getQlikSenseCloudAppInfo(appId); + + logger.verbose(`QLIK SENSE CLOUD: App info obtained. App ID="${appId}"`); + logger.debug(`QLIK SENSE CLOUD: App info: ${JSON.stringify(appInfo, null, 2)}`); + } catch (err) { + logger.error(`QLIK SENSE CLOUD: Could not get app info. Error=${JSON.stringify(err, null, 2)}`); + } + + // App metadata is available via "GET /v1/apps/{appId}/data/metadata" + // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-data-metadata + try { + appMetadata = await getQlikSenseCloudAppMetadata(appId); + + logger.verbose(`QLIK SENSE CLOUD: App metadata obtained. App ID="${appId}"`); + logger.debug(`QLIK SENSE CLOUD: App metadata: ${JSON.stringify(appMetadata, null, 2)}`); + } catch (err) { + logger.error(`QLIK SENSE CLOUD: Could not get app metadata. Error=${JSON.stringify(err, null, 2)}`); + } + + // App items are available via "GET /v1/items" + // https://qlik.dev/apis/rest/items/#get-v1-items + try { + appItems = await getQlikSenseCloudAppItems(appId); + + // There should be exactly one item in the appItems.data array, with a resourceId property that is the same as the app ID + // error if not + if (appItems?.data.length !== 1 || appItems?.data[0].resourceId !== appId) { + logger.error( + `QLIK SENSE CLOUD: App items obtained, but app ID does not match. App ID="${appId}", appItems="${JSON.stringify( + appItems, + null, + 2, + )}"`, + ); + + // Set appItems to empty object + appItems = {}; + } + + logger.verbose(`QLIK SENSE CLOUD: App items obtained. App ID="${appId}"`); + logger.debug(`QLIK SENSE CLOUD: App items: ${JSON.stringify(appItems, null, 2)}`); + } catch (err) { + logger.error(`QLIK SENSE CLOUD: Could not get app items. Error=${JSON.stringify(err, null, 2)}`); + } + + // Get info from config file + const tenantUrl = globals.config.get('Butler.qlikSenseCloud.event.mqtt.tenant.tenantUrl'); + + // Build URL to the app in Qlik Sense Cloud + // Format: /sense/app/ + // Take into account that tenant URL might have a trailing slash + const appUrl = `${tenantUrl}${tenantUrl.endsWith('/') ? '' : '/'}sense/app/${appId}`; + + // Save script log to disk file, if enabled + if (globals.config.get('Butler.scriptLog.storeOnDisk.qsCloud.appReloadFailure.enable') === true) { + logger.verbose(`QLIK SENSE CLOUD: Storing script log to disk file`); + + // Get path to the directory where script logs will be stored + const scriptLogDirRoot = globals.config.get('Butler.scriptLog.storeOnDisk.qsCloud.appReloadFailure.logDirectory'); + + // Create directory for script logs, if needed + // eventTime has format "2024-10-14T11:31:39Z" + // Set logDate variable to date part of eventTime + const logDate = eventTime.slice(0, 10); + const reloadLogDir = path.resolve(scriptLogDirRoot, logDate); + + logger.debug(`QLIK SENSE CLOUD: Script log directory: ${reloadLogDir}`); + + // Get error time stamp from eventTime, in format YYY-MM-DD_HH-MM-SS + const logTimeStamp = eventTime.slice(0, 19).replace(/ /g, '_').replace(/:/g, '-'); + + // Create directory for script logs, if needed + fs.mkdirSync(reloadLogDir, { recursive: true }); + + const fileName = path.resolve(reloadLogDir, `${logTimeStamp}_appId=${appId}_reloadId=${reloadId}.log`); + + // Write script log to disk file + try { + logger.info(`QLIK SENSE CLOUD: Writing failed task script log: ${fileName}`); + fs.writeFileSync(fileName, scriptLog.scriptLogFull.join('\n')); + } catch (err) { + logger.error(`QLIK SENSE CLOUD: Could not store script log to disk file. File="${fileName}", error=${err}`); + } + } + // Post to Teams when an app reload has failed, if enabled if ( globals.config.get('Butler.qlikSenseCloud.event.mqtt.tenant.alert.teamsNotification.reloadAppFailure.enable') === @@ -94,64 +224,18 @@ export async function handleQlikSenseCloudAppReloadFinished(message) { 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.teamsNotification.reloadAppFailure.basicContentOnly', ) === false ) { - // Get extended info about the event - // This includes: - // - Reload script log - // - Reload info - // - App info - - // Script log is available via "GET /v1/apps/{appId}/reloads/logs/{reloadId}" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-reloads-logs-reloadId - try { - const headLineCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.teamsNotification.reloadAppFailure.headScriptLogLines', - ); - - const tailLineCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.teamsNotification.reloadAppFailure.tailScriptLogLines', - ); - - scriptLog = await getQlikSenseCloudAppReloadScriptLog(appId, reloadId, headLineCount, tailLineCount); - - // If return value is false, the script log could not be obtained - if (scriptLog === false) { - logger.warn( - `QLIK SENSE CLOUD: Could not get app reload script log. App ID="${appId}", reload ID="${reloadId}"`, - ); - } else { - logger.verbose( - `QLIK SENSE CLOUD: App reload script log obtained. App ID="${appId}", reload ID="${reloadId}"`, - ); - } - logger.debug(`QLIK SENSE CLOUD: App reload script log: ${scriptLog}`); - } catch (err) { - logger.error( - `QLIK SENSE CLOUD: Could not get app reload script log. Error=${JSON.stringify(err, null, 2)}`, - ); - } - - // Reload info is available via "GET /v1/reloads/{reloadId}" - // https://qlik.dev/apis/rest/reloads/#get-v1-reloads-reloadId - try { - reloadInfo = await getQlikSenseCloudAppReloadInfo(reloadId); - reloadTrigger = reloadInfo.type; - - logger.verbose(`QLIK SENSE CLOUD: App reload info obtained. App ID="${appId}", reload ID="${reloadId}"`); - logger.debug(`QLIK SENSE CLOUD: App reload info: ${JSON.stringify(reloadInfo, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app reload info. Error=${JSON.stringify(err, null, 2)}`); - } - - // App info is available via "GET /v1/apps/{appId}" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId - try { - appInfo = await getQlikSenseCloudAppInfo(appId); - - logger.verbose(`QLIK SENSE CLOUD: App info obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App info: ${JSON.stringify(appInfo, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app info. Error=${JSON.stringify(err, null, 2)}`); - } + const headLineCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.teamsNotification.reloadAppFailure.headScriptLogLines', + ); + + const tailLineCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.teamsNotification.reloadAppFailure.tailScriptLogLines', + ); + + scriptLog.HeadCount = headLineCount; + scriptLog.TailCount = tailLineCount; + scriptLog.scriptLogHead = getQlikSenseCloudAppReloadScriptLogHead(scriptLog.scriptLogFull, headLineCount); + scriptLog.scriptLogTail = getQlikSenseCloudAppReloadScriptLogTail(scriptLog.scriptLogFull, tailLineCount); } else { // Use the basic info provided in the event/MQTT message scriptLog = {}; @@ -159,51 +243,6 @@ export async function handleQlikSenseCloudAppReloadFinished(message) { reloadInfo.reloadId = reloadId; } - // App metadata is available via "GET /v1/apps/{appId}/data/metadata" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-data-metadata - try { - appMetadata = await getQlikSenseCloudAppMetadata(appId); - - logger.verbose(`QLIK SENSE CLOUD: App metadata obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App metadata: ${JSON.stringify(appMetadata, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app metadata. Error=${JSON.stringify(err, null, 2)}`); - } - - // App items are available via "GET /v1/items" - // https://qlik.dev/apis/rest/items/#get-v1-items - try { - appItems = await getQlikSenseCloudAppItems(appId); - - // There should be exactly one item in the appItems.data array, with a resourceId property that is the same as the app ID - // error if not - if (appItems?.data.length !== 1 || appItems?.data[0].resourceId !== appId) { - logger.error( - `QLIK SENSE CLOUD: App items obtained, but app ID does not match. App ID="${appId}", appItems="${JSON.stringify( - appItems, - null, - 2, - )}"`, - ); - - // Set appItems to empty object - appItems = {}; - } - - logger.verbose(`QLIK SENSE CLOUD: App items obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App items: ${JSON.stringify(appItems, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app items. Error=${JSON.stringify(err, null, 2)}`); - } - - // Get info from config file - const tenantUrl = globals.config.get('Butler.qlikSenseCloud.event.mqtt.tenant.tenantUrl'); - - // Build URL to the app in Qlik Sense Cloud - // Format: /sense/app/ - // Take into account that tenant URL might have a trailing slash - const appUrl = `${tenantUrl}${tenantUrl.endsWith('/') ? '' : '/'}sense/app/${appId}`; - sendQlikSenseCloudAppReloadFailureNotificationTeams({ tenantId, tenantComment, @@ -257,64 +296,18 @@ export async function handleQlikSenseCloudAppReloadFinished(message) { 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.slackNotification.reloadAppFailure.basicContentOnly', ) === false ) { - // Get extended info about the event - // This includes: - // - Reload script log - // - Reload info - // - App info - - // Script log is available via "GET /v1/apps/{appId}/reloads/logs/{reloadId}" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-reloads-logs-reloadId - try { - const headLineCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.slackNotification.reloadAppFailure.headScriptLogLines', - ); - - const tailLineCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.slackNotification.reloadAppFailure.tailScriptLogLines', - ); - - scriptLog = await getQlikSenseCloudAppReloadScriptLog(appId, reloadId, headLineCount, tailLineCount); - - // If return value is false, the script log could not be obtained - if (scriptLog === false) { - logger.warn( - `QLIK SENSE CLOUD: Could not get app reload script log. App ID="${appId}", reload ID="${reloadId}"`, - ); - } else { - logger.verbose( - `QLIK SENSE CLOUD: App reload script log obtained. App ID="${appId}", reload ID="${reloadId}"`, - ); - } - logger.debug(`QLIK SENSE CLOUD: App reload script log: ${scriptLog}`); - } catch (err) { - logger.error( - `QLIK SENSE CLOUD: Could not get app reload script log. Error=${JSON.stringify(err, null, 2)}`, - ); - } - - // Reload info is available via "GET /v1/reloads/{reloadId}" - // https://qlik.dev/apis/rest/reloads/#get-v1-reloads-reloadId - try { - reloadInfo = await getQlikSenseCloudAppReloadInfo(reloadId); - reloadTrigger = reloadInfo.type; - - logger.verbose(`QLIK SENSE CLOUD: App reload info obtained. App ID="${appId}", reload ID="${reloadId}"`); - logger.debug(`QLIK SENSE CLOUD: App reload info: ${JSON.stringify(reloadInfo, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app reload info. Error=${JSON.stringify(err, null, 2)}`); - } - - // App info is available via "GET /v1/apps/{appId}" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId - try { - appInfo = await getQlikSenseCloudAppInfo(appId); - - logger.verbose(`QLIK SENSE CLOUD: App info obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App info: ${JSON.stringify(appInfo, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app info. Error=${JSON.stringify(err, null, 2)}`); - } + const headLineCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.slackNotification.reloadAppFailure.headScriptLogLines', + ); + + const tailLineCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.slackNotification.reloadAppFailure.tailScriptLogLines', + ); + + scriptLog.HeadCount = headLineCount; + scriptLog.TailCount = tailLineCount; + scriptLog.scriptLogHead = getQlikSenseCloudAppReloadScriptLogHead(scriptLog.scriptLogFull, headLineCount); + scriptLog.scriptLogTail = getQlikSenseCloudAppReloadScriptLogTail(scriptLog.scriptLogFull, tailLineCount); } else { // Use the basic info provided in the event/MQTT message scriptLog = {}; @@ -322,51 +315,6 @@ export async function handleQlikSenseCloudAppReloadFinished(message) { reloadInfo.reloadId = reloadId; } - // App metadata is available via "GET /v1/apps/{appId}/data/metadata" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-data-metadata - try { - appMetadata = await getQlikSenseCloudAppMetadata(appId); - - logger.verbose(`QLIK SENSE CLOUD: App metadata obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App metadata: ${JSON.stringify(appMetadata, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app metadata. Error=${JSON.stringify(err, null, 2)}`); - } - - // App items are available via "GET /v1/items" - // https://qlik.dev/apis/rest/items/#get-v1-items - try { - appItems = await getQlikSenseCloudAppItems(appId); - - // There should be exactly one item in the appItems.data array, with a resourceId property that is the same as the app ID - // error if not - if (appItems?.data.length !== 1 || appItems?.data[0].resourceId !== appId) { - logger.error( - `QLIK SENSE CLOUD: App items obtained, but app ID does not match. App ID="${appId}", appItems="${JSON.stringify( - appItems, - null, - 2, - )}"`, - ); - - // Set appItems to empty object - appItems = {}; - } - - logger.verbose(`QLIK SENSE CLOUD: App items obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App items: ${JSON.stringify(appItems, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app items. Error=${JSON.stringify(err, null, 2)}`); - } - - // Get info from config file - const tenantUrl = globals.config.get('Butler.qlikSenseCloud.event.mqtt.tenant.tenantUrl'); - - // Build URL to the app in Qlik Sense Cloud - // Format: /sense/app/ - // Take into account that tenant URL might have a trailing slash - const appUrl = `${tenantUrl}${tenantUrl.endsWith('/') ? '' : '/'}sense/app/${appId}`; - // Send Slack notification sendQlikSenseCloudAppReloadFailureNotificationSlack({ tenantId, @@ -421,101 +369,18 @@ export async function handleQlikSenseCloudAppReloadFinished(message) { // - App metadata // - App items - // Script log is available via "GET /v1/apps/{appId}/reloads/logs/{reloadId}" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-reloads-logs-reloadId - try { - const headLineCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.headScriptLogLines', - ); - - const tailLineCount = globals.config.get( - 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.tailScriptLogLines', - ); - - scriptLog = await getQlikSenseCloudAppReloadScriptLog(appId, reloadId, headLineCount, tailLineCount); - - // If return value is false, the script log could not be obtained - if (scriptLog === false) { - logger.warn( - `QLIK SENSE CLOUD: Could not get app reload script log. App ID="${appId}", reload ID="${reloadId}"`, - ); - } else { - logger.verbose( - `QLIK SENSE CLOUD: App reload script log obtained. App ID="${appId}", reload ID="${reloadId}"`, - ); - } - logger.debug(`QLIK SENSE CLOUD: App reload script log: ${scriptLog}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app reload script log. Error=${JSON.stringify(err, null, 2)}`); - } - - // Reload info is available via "GET /v1/reloads/{reloadId}" - // https://qlik.dev/apis/rest/reloads/#get-v1-reloads-reloadId - try { - reloadInfo = await getQlikSenseCloudAppReloadInfo(reloadId); - reloadTrigger = reloadInfo.type; - - logger.verbose(`QLIK SENSE CLOUD: App reload info obtained. App ID="${appId}", reload ID="${reloadId}"`); - logger.debug(`QLIK SENSE CLOUD: App reload info: ${JSON.stringify(reloadInfo, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app reload info. Error=${JSON.stringify(err, null, 2)}`); - } - - // App info is available via "GET /v1/apps/{appId}" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId - try { - appInfo = await getQlikSenseCloudAppInfo(appId); - - logger.verbose(`QLIK SENSE CLOUD: App info obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App info: ${JSON.stringify(appInfo, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app info. Error=${JSON.stringify(err, null, 2)}`); - } - - // App metadata is available via "GET /v1/apps/{appId}/data/metadata" - // https://qlik.dev/apis/rest/apps/#get-v1-apps-appId-data-metadata - try { - appMetadata = await getQlikSenseCloudAppMetadata(appId); - - logger.verbose(`QLIK SENSE CLOUD: App metadata obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App metadata: ${JSON.stringify(appMetadata, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app metadata. Error=${JSON.stringify(err, null, 2)}`); - } - - // App items are available via "GET /v1/items" - // https://qlik.dev/apis/rest/items/#get-v1-items - try { - appItems = await getQlikSenseCloudAppItems(appId); - - // There should be exactly one item in the appItems.data array, with a resourceId property that is the same as the app ID - // error if not - if (appItems?.data.length !== 1 || appItems?.data[0].resourceId !== appId) { - logger.error( - `QLIK SENSE CLOUD: App items obtained, but app ID does not match. App ID="${appId}", appItems="${JSON.stringify( - appItems, - null, - 2, - )}"`, - ); - - // Set appItems to empty object - appItems = {}; - } - - logger.verbose(`QLIK SENSE CLOUD: App items obtained. App ID="${appId}"`); - logger.debug(`QLIK SENSE CLOUD: App items: ${JSON.stringify(appItems, null, 2)}`); - } catch (err) { - logger.error(`QLIK SENSE CLOUD: Could not get app items. Error=${JSON.stringify(err, null, 2)}`); - } + const headLineCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.headScriptLogLines', + ); - // Get info from config file - const tenantUrl = globals.config.get('Butler.qlikSenseCloud.event.mqtt.tenant.tenantUrl'); + const tailLineCount = globals.config.get( + 'Butler.qlikSenseCloud.event.mqtt.tenant.alert.emailNotification.reloadAppFailure.tailScriptLogLines', + ); - // Build URL to the app in Qlik Sense Cloud - // Format: /sense/app/ - // Take into account that tenant URL might have a trailing slash - const appUrl = `${tenantUrl}${tenantUrl.endsWith('/') ? '' : '/'}sense/app/${appId}`; + scriptLog.HeadCount = headLineCount; + scriptLog.TailCount = tailLineCount; + scriptLog.scriptLogHead = getQlikSenseCloudAppReloadScriptLogHead(scriptLog.scriptLogFull, headLineCount); + scriptLog.scriptLogTail = getQlikSenseCloudAppReloadScriptLogTail(scriptLog.scriptLogFull, tailLineCount); // Send email notification sendQlikSenseCloudAppReloadFailureNotificationEmail({ diff --git a/src/lib/qscloud/slack_notification_qscloud.js b/src/lib/qscloud/slack_notification_qscloud.js index eb049989..362c611d 100644 --- a/src/lib/qscloud/slack_notification_qscloud.js +++ b/src/lib/qscloud/slack_notification_qscloud.js @@ -203,7 +203,7 @@ async function sendSlack(slackConfig, templateContext, msgType) { // Function to send Qlik Sense Cloud app reload failed alert export function sendQlikSenseCloudAppReloadFailureNotificationSlack(reloadParams) { rateLimiterMemoryFailedReloads - .consume(reloadParams.reloadId, 1) + .consume(reloadParams.appId, 1) .then(async (rateLimiterRes) => { try { globals.logger.info( diff --git a/src/lib/qseow/scriptlog.js b/src/lib/qseow/scriptlog.js index fb8bc55b..b2ae1e61 100644 --- a/src/lib/qseow/scriptlog.js +++ b/src/lib/qseow/scriptlog.js @@ -288,7 +288,7 @@ export async function getScriptLog(reloadTaskId, headLineCount, tailLineCount) { export async function failedTaskStoreLogOnDisk(reloadParams) { try { // Get top level directory where logs should be stored - const reloadLogDirRoot = globals.config.get('Butler.scriptLog.storeOnDisk.reloadTaskFailure.logDirectory'); + const reloadLogDirRoot = globals.config.get('Butler.scriptLog.storeOnDisk.clientManaged.reloadTaskFailure.logDirectory'); // Get misc script log info const { scriptLog } = reloadParams; diff --git a/src/udp/udp_handlers.js b/src/udp/udp_handlers.js index 04e9d77e..18416ce0 100644 --- a/src/udp/udp_handlers.js +++ b/src/udp/udp_handlers.js @@ -362,7 +362,7 @@ const schedulerFailed = async (msg) => { })); // Store script log to disk - if (globals.config.get('Butler.scriptLog.storeOnDisk.reloadTaskFailure.enable') === true) { + if (globals.config.get('Butler.scriptLog.storeOnDisk.clientManaged.reloadTaskFailure.enable') === true) { failedTaskStoreLogOnDisk({ hostName: msg[1], user: msg[4].replace(/\\/g, '/'),