diff --git a/README.md b/README.md index bada4ce..bdde941 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ You might want to store the webhook URL as a secret. ## Usage +### Results + To send the test results summary to Slack: ```sh @@ -54,6 +56,20 @@ npx slack-ctrf results /path/to/ctrf-report.json ![Results view](assets/results.png) +### AI Summary + +To send AI failed test summary to Slack: + +```sh +npx slack-ctrf ai /path/to/ctrf-report.json +``` + +![AI view](assets/ai.png) + +See the [AI Test Reporter](https://github.com/ctrf-io/ai-test-reporter) to add AI summaries to your CTRF report + +### Flaky + To send flaky test report to Slack: ```sh diff --git a/assets/ai.png b/assets/ai.png new file mode 100644 index 0000000..8ed1d52 Binary files /dev/null and b/assets/ai.png differ diff --git a/ctrf-report.json b/ctrf-report.json index 7f3124e..5215ff9 100644 --- a/ctrf-report.json +++ b/ctrf-report.json @@ -44,11 +44,12 @@ "duration": 1100 }, { - "name": "should fail to update profile on network failure", + "name": "should render title", "status": "failed", "duration": 900, "message": "Network Timeout", - "trace": "ProfileUpdateTest.js:60" + "trace": "ProfileUpdateTest.js:60", + "ai": "The test failed because the page title didn't match the expected value within the given timeout period. \n\nTo resolve this issue, you should first check if the title of the page is correct in your application. It seems there might be a typo or a misunderstanding about what the actual title should be. If \"Common Test Report Format\" is indeed the correct title, you'll need to update your test expectations. On the other hand, if \"Uncommon Test Report Format\" is the intended title, you'll need to fix the title in your application code.\n\nAnother possibility is that the page might be taking longer to load than expected, causing the title to not appear within the 5-second timeout. In this case, you could try increasing the timeout duration in your test to give the page more time to load completely." }, { "name": "should load user data", diff --git a/package-lock.json b/package-lock.json index c327432..29ec498 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,17 +1,20 @@ { "name": "slack-ctrf", - "version": "0.0.1", + "version": "0.0.13", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "slack-ctrf", - "version": "0.0.1", - "license": "ISC", + "version": "0.0.13", + "license": "MIT", "dependencies": { "typescript": "^5.4.5", "yargs": "^17.7.2" }, + "bin": { + "slack-ctrf": "dist/cli.js" + }, "devDependencies": { "@types/node": "^20.12.7", "@types/yargs": "^17.0.32", diff --git a/package.json b/package.json index ab22169..406bc43 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "slack-ctrf", - "version": "0.0.12", + "version": "0.0.13", "description": "", "main": "index.js", "scripts": { diff --git a/src/cli.ts b/src/cli.ts index 90fb2b0..667b892 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -2,7 +2,7 @@ import yargs from 'yargs/yargs'; import { hideBin } from 'yargs/helpers'; import { parseCtrfFile } from './ctrf-parser'; -import { formatResultsMessage, formatFailedTestsMessage, formatFlakyTestsMessage } from './message-formatter'; +import { formatResultsMessage, formatFailedTestsMessage, formatFlakyTestsMessage, formatAiTestSummary } from './message-formatter'; import { sendSlackMessage } from './slack-notify'; const argv = yargs(hideBin(process.argv)) @@ -102,5 +102,40 @@ const argv = yargs(hideBin(process.argv)) } } ) + .command( + 'ai ', + 'Send ai failed test summary to Slack', + (yargs) => { + return yargs.positional('path', { + describe: 'Path to the CTRF file', + type: 'string', + demandOption: true, + }) + .option('title', { + alias: 't', + type: 'string', + description: 'Title of notification', + default: "AI Summary", + }); + }, + async (argv) => { + try { + const ctrfData = parseCtrfFile(argv.path as string); + for (const test of ctrfData.results.tests) { + if (test.status === "failed") { + const message = formatAiTestSummary(test, ctrfData.results.environment, {title: argv.title}); + if (message) { + await sendSlackMessage(message); + console.log('AI test summary sent to Slack.'); + } else { + console.log('No AI summary detected. No message sent.'); + } + } + } + } catch (error: any) { + console.error('Error:', error.message); + } + } + ) .help() .argv; \ No newline at end of file diff --git a/src/message-formatter.ts b/src/message-formatter.ts index 93f39eb..7c71ae2 100644 --- a/src/message-formatter.ts +++ b/src/message-formatter.ts @@ -1,4 +1,4 @@ -import { CtrfReport } from '../types/ctrf'; +import { CtrfEnvironment, CtrfReport, CtrfTest } from '../types/ctrf'; type Options = { @@ -94,7 +94,7 @@ export const formatResultsMessage = (ctrf: CtrfReport, options?: Options): objec elements: [ { type: "mrkdwn", - text: "" + text: "" } ] }); @@ -194,13 +194,12 @@ export const formatFlakyTestsMessage = (ctrf: CtrfReport, options?: Options): ob }); } - // Add link to plugin documentation or repository blocks.push({ type: "context", elements: [ { type: "mrkdwn", - text: "" + text: "" } ] }); @@ -214,3 +213,105 @@ export const formatFlakyTestsMessage = (ctrf: CtrfReport, options?: Options): ob ] }; }; + + export const formatAiTestSummary = (test: CtrfTest, environment: CtrfEnvironment | undefined, options?: Options): object | null => { + const { name, ai, status } = test + + if (!ai || status === "passed") { return null} + + let title = options?.title ? options?.title : `AI Test summary`; + let missingEnvProperties: string[] = []; + + let buildInfo = "*Build:* No build information provided"; + if (environment) { + const { buildName, buildNumber, buildUrl } = environment; + + if (buildName && buildNumber) { + const buildText = buildUrl ? `<${buildUrl}|${buildName} #${buildNumber}>` : `${buildName} #${buildNumber}`; + buildInfo = `*Build:* ${buildText}`; + } else if (buildName || buildNumber) { + buildInfo = `*Build:* ${buildName || ''} ${buildNumber || ''}`; + } + + if (!buildName) { + missingEnvProperties.push('buildName'); + } + + if (!buildNumber) { + missingEnvProperties.push('buildNumber'); + } + + if (!buildUrl) { + missingEnvProperties.push('buildUrl'); + } + } else { + missingEnvProperties = ['buildName', 'buildNumber', 'buildUrl']; + } + + const color = '#800080' + const resultText = `*Status:* Failed` + + const aiSummaryText = `*:sparkles: AI Summary:* ${ai}`; + + const blocks: any[] = [ + { + type: "header", + text: { + type: "plain_text", + text: title, + emoji: true + } + }, + { + type: "section", + text: { + type: "mrkdwn", + text: `*Test Name:* ${name}\n${resultText}` + } + }, + { + type: "section", + text: { + type: "mrkdwn", + text: `${aiSummaryText}` + } + }, + { + type: "section", + text: { + type: "mrkdwn", + text: `${buildInfo}` + } + } + ]; + + if (missingEnvProperties.length > 0) { + blocks.push({ + type: "section", + text: { + type: "mrkdwn", + text: `:warning: Missing environment properties: ${missingEnvProperties.join(', ')}. Add these to your test for a better experience.` + } + }); + } + + blocks.push({ + type: "context", + elements: [ + { + type: "mrkdwn", + text: "" + } + ] + }); + + return { + attachments: [ + { + color: color, + blocks: blocks + } + ] + }; + }; + diff --git a/types/ctrf.d.ts b/types/ctrf.d.ts index 61a72c6..9f4b650 100644 --- a/types/ctrf.d.ts +++ b/types/ctrf.d.ts @@ -32,6 +32,7 @@ export interface CtrfTest { suite?: string message?: string trace?: string + ai?: string rawStatus?: string tags?: string[] type?: string