feat: failed method

ctrf-io · Oct 22, 2024 · 7ed800a · 7ed800a
1 parent 70e96ec
commit 7ed800a
Show file tree

Hide file tree

Showing 7 changed files with 324 additions and 23 deletions.
diff --git a/ctrf-report.json b/ctrf-report.json
@@ -56,6 +56,14 @@
         "status": "pending",
         "duration": 0
       },
+      {
+        "name": "should display title",
+        "status": "failed",
+        "duration": 800,
+        "message": "\u001b[31mTimed out 5000ms waiting for \u001b[39m\u001b[2mexpect(\u001b[22m\u001b[31mlocator\u001b[39m\u001b[2m).\u001b[22mtoHaveTitle\u001b[2m(\u001b[22m\u001b[32mexpected\u001b[39m\u001b[2m)\u001b[22m\n\nLocator: locator(':root')\nExpected pattern: \u001b[32m/Playwrc cight/\u001b[39m\nReceived string:  \u001b[31m\"Fast and reliable end-to-end testing for modern web apps | Playwright\"\u001b[39m\nCall log:\n  \u001b[2m- expect.toHaveTitle with timeout 5000ms\u001b[22m\n\u001b[2m  - waiting for locator(':root')\u001b[22m\n\u001b[2m  -   locator resolved to <html lang=\"en\" dir=\"ltr\" data-theme=\"light\" data-has-…>…</html>\u001b[22m\n\u001b[2m  -   unexpected value \"Fast and reliable end-to-end testing for modern web apps | Playwright\"\u001b[22m\n",
+        "trace": "ProfileTest.js:45",
+        "ai": "The test failed because the page title didn't match the expected value within the given timeout period.\n\nTo resolve this issue, you should first check if the title of the page is correct in your application. It seems there might be a typo or a misunderstanding about what the actual title should be. If 'Common Test Report Format' is indeed the correct title, you'll need to update your test expectations. On the other hand, if 'Uncommon Test Report Format' is the intended title, you'll need to fix the title in your application code.\n\nAnother possibility is that the page might be taking longer to load than expected, causing the title to not appear within the 5-second timeout. In this case, you could try increasing the timeout duration in your test to give the page more time to load completely."
+      },
       {
         "name": "should handle session timeouts",
         "status": "passed",

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,15 +1,15 @@
 {
   "name": "slack-ctrf",
-  "version": "0.0.14",
-  "description": "",
+  "version": "0.0.15",
+  "description": "Send Slack notifications with test results from popular testing frameworks using a single command in your CI/CD",
   "main": "index.js",
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "bin": {
     "slack-ctrf": "./dist/cli.js"
   },
-  "repository": "github:ctrf-io/slack-ctrf",
+  "repository": "github:ctrf-io/slack-test-reporter",
   "files": [
     "dist/",
     "README.md"

diff --git a/src/cli.ts b/src/cli.ts
@@ -2,8 +2,9 @@
 import yargs from 'yargs/yargs';
 import { hideBin } from 'yargs/helpers';
 import { parseCtrfFile } from './ctrf-parser';
-import { formatResultsMessage, formatFailedTestsMessage, formatFlakyTestsMessage, formatAiTestSummary, formatConsolidatedAiTestSummary } from './message-formatter';
+import { formatResultsMessage, formatFlakyTestsMessage, formatAiTestSummary, formatConsolidatedAiTestSummary, formatConsolidatedFailedTestSummary, formatFailedTestSummary } from './message-formatter';
 import { sendSlackMessage } from './slack-notify';
+import { stripAnsiFromErrors } from './utils/common';
 
 const argv = yargs(hideBin(process.argv))
   .command(
@@ -45,7 +46,7 @@ const argv = yargs(hideBin(process.argv))
     }
   )
   .command(
-    'fail-details <path>',
+    'failed <path>',
     'Send failed test results to Slack',
     (yargs) => {
       return yargs.positional('path', {
@@ -57,20 +58,43 @@ const argv = yargs(hideBin(process.argv))
         alias: 't',
         type: 'string',
         description: 'Title of notification',
-        default: "Failed Tests",
+        default: ":x: Failed Test",
+      })
+      .option('consolidated', {
+        alias: 'c',
+        type: 'boolean',
+        description: 'Consolidate all failure summaries into a single message',
+        default: false,
       });
     },
     async (argv) => {
       try {
-        const ctrfData = parseCtrfFile(argv.path as string);
-        const message = formatFailedTestsMessage(ctrfData, {title: argv.title});
-        // await sendSlackMessage(message);
-        console.log('Coming soon!');
+        const ctrfData = stripAnsiFromErrors(parseCtrfFile(argv.path as string))
+        if (argv.consolidated) {
+            const message = formatConsolidatedFailedTestSummary(ctrfData.results.tests, ctrfData.results.environment, {title: argv.title})
+            if (message) {
+              await sendSlackMessage(message);
+              console.log('Failed test summary sent to Slack.');
+            } else {
+              console.log('No failed test summary detected. No message sent.');
+            }
+        } else {
+        for (const test of ctrfData.results.tests) {
+          if (test.status === "failed") {
+            const message = formatFailedTestSummary(test, ctrfData.results.environment, {title: argv.title});
+            if (message) {
+              await sendSlackMessage(message);
+              console.log('Failed test summary sent to Slack.');
+            } else {
+              console.log('No failed test summary detected. No message sent');
+            }
+          }
+        }
+      }
       } catch (error: any) {
         console.error('Error:', error.message);
       }
-    }
-  )
+    })
   .command(
     'flaky <path>',
     'Send flaky test results to Slack',

diff --git a/src/message-formatter.ts b/src/message-formatter.ts
@@ -110,15 +110,6 @@ export const formatResultsMessage = (ctrf: CtrfReport, options?: Options): objec
 };
 
 
-export const formatFailedTestsMessage = (ctrf: CtrfReport, options?: Options): string => {
-  const failedTests = ctrf.results.tests.filter(test => test.status === 'failed');
-  if (failedTests.length === 0) return 'No failed tests.';
-
-  let title = options?.title ? options?.title : "Failed Tests";
-
-  const message = failedTests.map(test => `Test: ${test.name}\nMessage: ${test.message}\n`).join('\n');
-  return `Failed Tests:\n${message}`;
-};
 
 export const formatFlakyTestsMessage = (ctrf: CtrfReport, options?: Options): object | null => {
   const { summary, environment, tests } = ctrf.results;
@@ -436,7 +427,237 @@ export const formatConsolidatedAiTestSummary = (
   };
 };
 
+export const formatConsolidatedFailedTestSummary = (
+  tests: CtrfTest[],
+  environment: CtrfEnvironment | undefined,
+  options?: Options
+): object | null => {
+
+  const color = '#FF0000';
+  const MAX_FAILED_TESTS = 20;
+  const charLimit = 2950;
+  const trimmedNotice = "\n:warning: Message trimmed as too long for Slack";
+  const failedTests = tests.filter(test => test.status === "failed");
+
+  if (failedTests.length === 0) {
+    return null;
+  }
+
+  const title = options?.title ? options.title : `:x: Failed Test Report`;
+
+  let buildInfo = "*Build:* No build information provided";
+  if (environment) {
+    const { buildName, buildNumber, buildUrl } = environment;
+    if (buildName && buildNumber) {
+      const buildText = buildUrl ? `<${buildUrl}|${buildName} #${buildNumber}>` : `${buildName} #${buildNumber}`;
+      buildInfo = `*Build:* ${buildText}`;
+    }
+  }
+
+  let missingEnvProperties: string[] = [];
+  if (!environment?.buildName) missingEnvProperties.push('buildName');
+  if (!environment?.buildNumber) missingEnvProperties.push('buildNumber');
+  if (!environment?.buildUrl) missingEnvProperties.push('buildUrl');
+
+  const blocks: any[] = [
+    {
+      type: "header",
+      text: {
+        type: "plain_text",
+        text: title,
+        emoji: true
+      }
+    },
+    {
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: buildInfo
+      }
+    },
+    {
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `*Total Failed Tests:* ${failedTests.length}`
+      }
+    },
+    {
+      type: "divider"
+    }
+  ];
+
+  const limitedFailedTests = failedTests.slice(0, MAX_FAILED_TESTS);
+
+  limitedFailedTests.forEach(test => {
+    const failSummary = test.message && test.message.length > charLimit
+      ? test.message.substring(0, charLimit - trimmedNotice.length) + trimmedNotice
+      : test.message || "No message provided";
+
+    blocks.push({
+      type: "header",
+      text: {
+        type: "plain_text",
+        text: `:x: ${test.name}`,
+        emoji: true
+      }
+    });
+
+    blocks.push({
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `${failSummary}`
+      }
+    });
+  });
+
+  if (missingEnvProperties.length > 0) {
+    blocks.push({
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `:warning: *Missing Environment Properties:* ${missingEnvProperties.join(', ')}. Please add these to your test configuration for a better experience.`
+      }
+    });
+  }
+
+  if (failedTests.length > MAX_FAILED_TESTS) {
+    blocks.push({
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `:information_source: Only the first ${MAX_FAILED_TESTS} failed tests are displayed. ${failedTests.length - MAX_FAILED_TESTS} additional failed tests were not included.`
+      }
+    });
+  }
+
+  blocks.push({
+    type: "context",
+    elements: [
+      {
+        type: "mrkdwn",
+        text: "<https://github.com/ctrf-io/slack-ctrf|Slack CTRF Test Reporter>"
+      }
+    ]
+  });
+
+  return {
+    attachments: [
+      {
+        color: color,
+        blocks: blocks
+      }
+    ]
+  };
+};
+
+export const formatFailedTestSummary = (test: CtrfTest, environment: CtrfEnvironment | undefined, options?: Options): object | null => {
+  const { name, message, status } = test;
+  const charLimit = 2950;
+  const trimmedNotice = "\n:warning: Message trimmed as too long for Slack";
+  const color = '#FF0000';
 
+  if (status !== "failed") {
+    return null;
+  }
+
+  let title = options?.title ? options?.title : `Failed Test summary`;
+  let missingEnvProperties: string[] = [];
+
+  let buildInfo = "*Build:* No build information provided";
+  if (environment) {
+    const { buildName, buildNumber, buildUrl } = environment;
+
+    if (buildName && buildNumber) {
+      const buildText = buildUrl ? `<${buildUrl}|${buildName} #${buildNumber}>` : `${buildName} #${buildNumber}`;
+      buildInfo = `*Build:* ${buildText}`;
+    } else if (buildName || buildNumber) {
+      buildInfo = `*Build:* ${buildName || ''} ${buildNumber || ''}`;
+    }
+
+    if (!buildName) {
+      missingEnvProperties.push('buildName');
+    }
+
+    if (!buildNumber) {
+      missingEnvProperties.push('buildNumber');
+    }
+
+    if (!buildUrl) {
+      missingEnvProperties.push('buildUrl');
+    }
+  } else {
+    missingEnvProperties = ['buildName', 'buildNumber', 'buildUrl'];
+  }
+
+  const enrichedMessage = message && message.length > charLimit
+    ? message.substring(0, charLimit - trimmedNotice.length) + trimmedNotice
+    : (message || "No message provided");
+
+  const failSummaryText = `*Message:* ${enrichedMessage}`;
+
+  const blocks: any[] = [
+    {
+      type: "header",
+      text: {
+        type: "plain_text",
+        text: title,
+        emoji: true
+      }
+    },
+    {
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `*Test Name:* ${name}`
+      }
+    },
+    {
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `${failSummaryText}`
+      }
+    },
+    {
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `${buildInfo}`
+      }
+    }
+  ];
+
+  if (missingEnvProperties.length > 0) {
+    blocks.push({
+      type: "section",
+      text: {
+        type: "mrkdwn",
+        text: `:warning: Missing environment properties: ${missingEnvProperties.join(', ')}. Add these to your test for a better experience.`
+      }
+    });
+  }
+
+  blocks.push({
+    type: "context",
+    elements: [
+      {
+        type: "mrkdwn",
+        text: "<https://github.com/ctrf-io/slack-ctrf|Slack CTRF Test Reporter>"
+      }
+    ]
+  });
+
+  return {
+    attachments: [
+      {
+        color: color,
+        blocks: blocks
+      }
+    ]
+  };
+};