Merge pull request #8 from ctrf-io/feat/ai-summary

feat: add ai summary

README.md

@@ -46,6 +46,8 @@ You might want to store the webhook URL as a secret.
 
 ## Usage
 
+### Results
+
 To send the test results summary to Slack:
 
 ```sh
@@ -54,6 +56,20 @@ npx slack-ctrf results /path/to/ctrf-report.json
 
 ![Results view](assets/results.png)
 
+### AI Summary
+
+To send AI failed test summary to Slack:
+
+```sh
+npx slack-ctrf ai /path/to/ctrf-report.json
+```
+
+![AI view](assets/ai.png)
+
+See the [AI Test Reporter](https://github.com/ctrf-io/ai-test-reporter) to add AI summaries to your CTRF report
+
+### Flaky
+
 To send flaky test report to Slack:
 
 ```sh

ctrf-report.json

@@ -44,11 +44,12 @@
         "duration": 1100
       },
       {
-        "name": "should fail to update profile on network failure",
+        "name": "should render title",
         "status": "failed",
         "duration": 900,
         "message": "Network Timeout",
-        "trace": "ProfileUpdateTest.js:60"
+        "trace": "ProfileUpdateTest.js:60",
+        "ai": "The test failed because the page title didn't match the expected value within the given timeout period. \n\nTo resolve this issue, you should first check if the title of the page is correct in your application. It seems there might be a typo or a misunderstanding about what the actual title should be. If \"Common Test Report Format\" is indeed the correct title, you'll need to update your test expectations. On the other hand, if \"Uncommon Test Report Format\" is the intended title, you'll need to fix the title in your application code.\n\nAnother possibility is that the page might be taking longer to load than expected, causing the title to not appear within the 5-second timeout. In this case, you could try increasing the timeout duration in your test to give the page more time to load completely."
       },
       {
         "name": "should load user data",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "slack-ctrf",
-  "version": "0.0.12",
+  "version": "0.0.13",
   "description": "",
   "main": "index.js",
   "scripts": {

src/cli.ts

@@ -2,7 +2,7 @@
 import yargs from 'yargs/yargs';
 import { hideBin } from 'yargs/helpers';
 import { parseCtrfFile } from './ctrf-parser';
-import { formatResultsMessage, formatFailedTestsMessage, formatFlakyTestsMessage } from './message-formatter';
+import { formatResultsMessage, formatFailedTestsMessage, formatFlakyTestsMessage, formatAiTestSummary } from './message-formatter';
 import { sendSlackMessage } from './slack-notify';
 
 const argv = yargs(hideBin(process.argv))
@@ -102,5 +102,40 @@ const argv = yargs(hideBin(process.argv))
       }
     }
   )
+  .command(
+    'ai <path>',
+    'Send ai failed test summary to Slack',
+    (yargs) => {
+      return yargs.positional('path', {
+        describe: 'Path to the CTRF file',
+        type: 'string',
+        demandOption: true,
+      })
+      .option('title', {
+        alias: 't',
+        type: 'string',
+        description: 'Title of notification',
+        default: "AI Summary",
+      });
+    },
+    async (argv) => {
+      try {
+        const ctrfData = parseCtrfFile(argv.path as string);
+        for (const test of ctrfData.results.tests) {
+          if (test.status === "failed") {
+            const message = formatAiTestSummary(test, ctrfData.results.environment, {title: argv.title});
+            if (message) {
+              await sendSlackMessage(message);
+              console.log('AI test summary sent to Slack.');
+            } else {
+              console.log('No AI summary detected. No message sent.');
+            }
+          }
+        }
+      } catch (error: any) {
+        console.error('Error:', error.message);
+      }
+    }
+  )
   .help()
   .argv;

src/message-formatter.ts

@@ -1,4 +1,4 @@
-import { CtrfReport } from '../types/ctrf';
+import { CtrfEnvironment, CtrfReport, CtrfTest } from '../types/ctrf';
 
 type Options = 
 {
@@ -94,7 +94,7 @@ export const formatResultsMessage = (ctrf: CtrfReport, options?: Options): objec
       elements: [
         {
           type: "mrkdwn",
-          text: "<https://github.com/ctrf-io/slack-ctrf|a CTRF plugin>"
+          text: "<https://github.com/ctrf-io/slack-ctrf|Slack CTRF Test Reporter>"
         }
       ]
     });
@@ -194,13 +194,12 @@ export const formatFlakyTestsMessage = (ctrf: CtrfReport, options?: Options): ob
       });
     }
 
-    // Add link to plugin documentation or repository
     blocks.push({
       type: "context",
       elements: [
         {
           type: "mrkdwn",
-          text: "<https://github.com/ctrf-io/slack-ctrf|a CTRF plugin>"
+          text: "<https://github.com/ctrf-io/slack-ctrf|Slack CTRF Test Reporter>"
         }
       ]
     });
@@ -214,3 +213,105 @@ export const formatFlakyTestsMessage = (ctrf: CtrfReport, options?: Options): ob
       ]
     };
   };
+
+  export const formatAiTestSummary = (test: CtrfTest, environment: CtrfEnvironment | undefined, options?: Options):  object | null => {
+    const { name, ai, status } = test
+
+    if (!ai || status === "passed") { return null}
+
+    let title = options?.title ? options?.title : `AI Test summary`;
+    let missingEnvProperties: string[] = [];
+
+    let buildInfo = "*Build:* No build information provided";
+    if (environment) {
+      const { buildName, buildNumber, buildUrl } = environment;
+
+      if (buildName && buildNumber) {
+        const buildText = buildUrl ? `<${buildUrl}|${buildName} #${buildNumber}>` : `${buildName} #${buildNumber}`;
+        buildInfo = `*Build:* ${buildText}`;
+      } else if (buildName || buildNumber) {
+        buildInfo = `*Build:* ${buildName || ''} ${buildNumber || ''}`;
+      }
+
+      if (!buildName) {
+        missingEnvProperties.push('buildName');
+      }
+
+      if (!buildNumber) {
+        missingEnvProperties.push('buildNumber');
+      }
+
+      if (!buildUrl) {
+        missingEnvProperties.push('buildUrl');
+      }
+    } else {
+      missingEnvProperties = ['buildName', 'buildNumber', 'buildUrl'];
+    }
+
+    const color = '#800080' 
+    const resultText = `*Status:* Failed`
+
+    const aiSummaryText = `*:sparkles: AI Summary:* ${ai}`;
+
+    const blocks: any[] = [
+      {
+        type: "header",
+        text: {
+          type: "plain_text",
+          text: title,
+          emoji: true
+        }
+      },
+      {
+        type: "section",
+        text: {
+          type: "mrkdwn",
+          text: `*Test Name:* ${name}\n${resultText}`
+        }
+      },
+      {
+        type: "section",
+        text: {
+          type: "mrkdwn",
+          text: `${aiSummaryText}`
+        }
+      },
+      {
+        type: "section",
+        text: {
+          type: "mrkdwn",
+          text: `${buildInfo}`
+        }
+      }
+    ];
+
+    if (missingEnvProperties.length > 0) {
+      blocks.push({
+        type: "section",
+        text: {
+          type: "mrkdwn",
+          text: `:warning: Missing environment properties: ${missingEnvProperties.join(', ')}. Add these to your test for a better experience.`
+        }
+      });
+    }
+
+    blocks.push({
+      type: "context",
+      elements: [
+        {
+          type: "mrkdwn",
+          text: "<https://github.com/ctrf-io/slack-ctrf|Slack CTRF Test Reporter>"
+        }
+      ]
+    });
+
+    return {
+      attachments: [
+        {
+          color: color,
+          blocks: blocks
+        }
+      ]
+    };
+  };
+

types/ctrf.d.ts

@@ -32,6 +32,7 @@ export interface CtrfTest {
   suite?: string
   message?: string
   trace?: string
+  ai?: string
   rawStatus?: string
   tags?: string[]
   type?: string