Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: failed method #14

Merged
merged 1 commit into from
Oct 22, 2024
Merged

feat: failed method #14

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions ctrf-report.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,14 @@
"status": "pending",
"duration": 0
},
{
"name": "should display title",
"status": "failed",
"duration": 800,
"message": "\u001b[31mTimed out 5000ms waiting for \u001b[39m\u001b[2mexpect(\u001b[22m\u001b[31mlocator\u001b[39m\u001b[2m).\u001b[22mtoHaveTitle\u001b[2m(\u001b[22m\u001b[32mexpected\u001b[39m\u001b[2m)\u001b[22m\n\nLocator: locator(':root')\nExpected pattern: \u001b[32m/Playwrc cight/\u001b[39m\nReceived string: \u001b[31m\"Fast and reliable end-to-end testing for modern web apps | Playwright\"\u001b[39m\nCall log:\n \u001b[2m- expect.toHaveTitle with timeout 5000ms\u001b[22m\n\u001b[2m - waiting for locator(':root')\u001b[22m\n\u001b[2m - locator resolved to <html lang=\"en\" dir=\"ltr\" data-theme=\"light\" data-has-…>…</html>\u001b[22m\n\u001b[2m - unexpected value \"Fast and reliable end-to-end testing for modern web apps | Playwright\"\u001b[22m\n",
"trace": "ProfileTest.js:45",
"ai": "The test failed because the page title didn't match the expected value within the given timeout period.\n\nTo resolve this issue, you should first check if the title of the page is correct in your application. It seems there might be a typo or a misunderstanding about what the actual title should be. If 'Common Test Report Format' is indeed the correct title, you'll need to update your test expectations. On the other hand, if 'Uncommon Test Report Format' is the intended title, you'll need to fix the title in your application code.\n\nAnother possibility is that the page might be taking longer to load than expected, causing the title to not appear within the 5-second timeout. In this case, you could try increasing the timeout duration in your test to give the page more time to load completely."
},
{
"name": "should handle session timeouts",
"status": "passed",
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 3 additions & 3 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
{
"name": "slack-ctrf",
"version": "0.0.14",
"description": "",
"version": "0.0.15",
"description": "Send Slack notifications with test results from popular testing frameworks using a single command in your CI/CD",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"bin": {
"slack-ctrf": "./dist/cli.js"
},
"repository": "github:ctrf-io/slack-ctrf",
"repository": "github:ctrf-io/slack-test-reporter",
"files": [
"dist/",
"README.md"
Expand Down
42 changes: 33 additions & 9 deletions src/cli.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@
import yargs from 'yargs/yargs';
import { hideBin } from 'yargs/helpers';
import { parseCtrfFile } from './ctrf-parser';
import { formatResultsMessage, formatFailedTestsMessage, formatFlakyTestsMessage, formatAiTestSummary, formatConsolidatedAiTestSummary } from './message-formatter';
import { formatResultsMessage, formatFlakyTestsMessage, formatAiTestSummary, formatConsolidatedAiTestSummary, formatConsolidatedFailedTestSummary, formatFailedTestSummary } from './message-formatter';
import { sendSlackMessage } from './slack-notify';
import { stripAnsiFromErrors } from './utils/common';

const argv = yargs(hideBin(process.argv))
.command(
Expand Down Expand Up @@ -45,7 +46,7 @@ const argv = yargs(hideBin(process.argv))
}
)
.command(
'fail-details <path>',
'failed <path>',
'Send failed test results to Slack',
(yargs) => {
return yargs.positional('path', {
Expand All @@ -57,20 +58,43 @@ const argv = yargs(hideBin(process.argv))
alias: 't',
type: 'string',
description: 'Title of notification',
default: "Failed Tests",
default: ":x: Failed Test",
})
.option('consolidated', {
alias: 'c',
type: 'boolean',
description: 'Consolidate all failure summaries into a single message',
default: false,
});
},
async (argv) => {
try {
const ctrfData = parseCtrfFile(argv.path as string);
const message = formatFailedTestsMessage(ctrfData, {title: argv.title});
// await sendSlackMessage(message);
console.log('Coming soon!');
const ctrfData = stripAnsiFromErrors(parseCtrfFile(argv.path as string))
if (argv.consolidated) {
const message = formatConsolidatedFailedTestSummary(ctrfData.results.tests, ctrfData.results.environment, {title: argv.title})
if (message) {
await sendSlackMessage(message);
console.log('Failed test summary sent to Slack.');
} else {
console.log('No failed test summary detected. No message sent.');
}
} else {
for (const test of ctrfData.results.tests) {
if (test.status === "failed") {
const message = formatFailedTestSummary(test, ctrfData.results.environment, {title: argv.title});
if (message) {
await sendSlackMessage(message);
console.log('Failed test summary sent to Slack.');
} else {
console.log('No failed test summary detected. No message sent');
}
}
}
}
} catch (error: any) {
console.error('Error:', error.message);
}
}
)
})
.command(
'flaky <path>',
'Send flaky test results to Slack',
Expand Down
239 changes: 230 additions & 9 deletions src/message-formatter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,15 +110,6 @@ export const formatResultsMessage = (ctrf: CtrfReport, options?: Options): objec
};


export const formatFailedTestsMessage = (ctrf: CtrfReport, options?: Options): string => {
const failedTests = ctrf.results.tests.filter(test => test.status === 'failed');
if (failedTests.length === 0) return 'No failed tests.';

let title = options?.title ? options?.title : "Failed Tests";

const message = failedTests.map(test => `Test: ${test.name}\nMessage: ${test.message}\n`).join('\n');
return `Failed Tests:\n${message}`;
};

export const formatFlakyTestsMessage = (ctrf: CtrfReport, options?: Options): object | null => {
const { summary, environment, tests } = ctrf.results;
Expand Down Expand Up @@ -436,7 +427,237 @@ export const formatConsolidatedAiTestSummary = (
};
};

export const formatConsolidatedFailedTestSummary = (
tests: CtrfTest[],
environment: CtrfEnvironment | undefined,
options?: Options
): object | null => {

const color = '#FF0000';
const MAX_FAILED_TESTS = 20;
const charLimit = 2950;
const trimmedNotice = "\n:warning: Message trimmed as too long for Slack";
const failedTests = tests.filter(test => test.status === "failed");

if (failedTests.length === 0) {
return null;
}

const title = options?.title ? options.title : `:x: Failed Test Report`;

let buildInfo = "*Build:* No build information provided";
if (environment) {
const { buildName, buildNumber, buildUrl } = environment;
if (buildName && buildNumber) {
const buildText = buildUrl ? `<${buildUrl}|${buildName} #${buildNumber}>` : `${buildName} #${buildNumber}`;
buildInfo = `*Build:* ${buildText}`;
}
}

let missingEnvProperties: string[] = [];
if (!environment?.buildName) missingEnvProperties.push('buildName');
if (!environment?.buildNumber) missingEnvProperties.push('buildNumber');
if (!environment?.buildUrl) missingEnvProperties.push('buildUrl');

const blocks: any[] = [
{
type: "header",
text: {
type: "plain_text",
text: title,
emoji: true
}
},
{
type: "section",
text: {
type: "mrkdwn",
text: buildInfo
}
},
{
type: "section",
text: {
type: "mrkdwn",
text: `*Total Failed Tests:* ${failedTests.length}`
}
},
{
type: "divider"
}
];

const limitedFailedTests = failedTests.slice(0, MAX_FAILED_TESTS);

limitedFailedTests.forEach(test => {
const failSummary = test.message && test.message.length > charLimit
? test.message.substring(0, charLimit - trimmedNotice.length) + trimmedNotice
: test.message || "No message provided";

blocks.push({
type: "header",
text: {
type: "plain_text",
text: `:x: ${test.name}`,
emoji: true
}
});

blocks.push({
type: "section",
text: {
type: "mrkdwn",
text: `${failSummary}`
}
});
});

if (missingEnvProperties.length > 0) {
blocks.push({
type: "section",
text: {
type: "mrkdwn",
text: `:warning: *Missing Environment Properties:* ${missingEnvProperties.join(', ')}. Please add these to your test configuration for a better experience.`
}
});
}

if (failedTests.length > MAX_FAILED_TESTS) {
blocks.push({
type: "section",
text: {
type: "mrkdwn",
text: `:information_source: Only the first ${MAX_FAILED_TESTS} failed tests are displayed. ${failedTests.length - MAX_FAILED_TESTS} additional failed tests were not included.`
}
});
}

blocks.push({
type: "context",
elements: [
{
type: "mrkdwn",
text: "<https://github.com/ctrf-io/slack-ctrf|Slack CTRF Test Reporter>"
}
]
});

return {
attachments: [
{
color: color,
blocks: blocks
}
]
};
};

export const formatFailedTestSummary = (test: CtrfTest, environment: CtrfEnvironment | undefined, options?: Options): object | null => {
const { name, message, status } = test;
const charLimit = 2950;
const trimmedNotice = "\n:warning: Message trimmed as too long for Slack";
const color = '#FF0000';

if (status !== "failed") {
return null;
}

let title = options?.title ? options?.title : `Failed Test summary`;
let missingEnvProperties: string[] = [];

let buildInfo = "*Build:* No build information provided";
if (environment) {
const { buildName, buildNumber, buildUrl } = environment;

if (buildName && buildNumber) {
const buildText = buildUrl ? `<${buildUrl}|${buildName} #${buildNumber}>` : `${buildName} #${buildNumber}`;
buildInfo = `*Build:* ${buildText}`;
} else if (buildName || buildNumber) {
buildInfo = `*Build:* ${buildName || ''} ${buildNumber || ''}`;
}

if (!buildName) {
missingEnvProperties.push('buildName');
}

if (!buildNumber) {
missingEnvProperties.push('buildNumber');
}

if (!buildUrl) {
missingEnvProperties.push('buildUrl');
}
} else {
missingEnvProperties = ['buildName', 'buildNumber', 'buildUrl'];
}

const enrichedMessage = message && message.length > charLimit
? message.substring(0, charLimit - trimmedNotice.length) + trimmedNotice
: (message || "No message provided");

const failSummaryText = `*Message:* ${enrichedMessage}`;

const blocks: any[] = [
{
type: "header",
text: {
type: "plain_text",
text: title,
emoji: true
}
},
{
type: "section",
text: {
type: "mrkdwn",
text: `*Test Name:* ${name}`
}
},
{
type: "section",
text: {
type: "mrkdwn",
text: `${failSummaryText}`
}
},
{
type: "section",
text: {
type: "mrkdwn",
text: `${buildInfo}`
}
}
];

if (missingEnvProperties.length > 0) {
blocks.push({
type: "section",
text: {
type: "mrkdwn",
text: `:warning: Missing environment properties: ${missingEnvProperties.join(', ')}. Add these to your test for a better experience.`
}
});
}

blocks.push({
type: "context",
elements: [
{
type: "mrkdwn",
text: "<https://github.com/ctrf-io/slack-ctrf|Slack CTRF Test Reporter>"
}
]
});

return {
attachments: [
{
color: color,
blocks: blocks
}
]
};
};



Expand Down
Loading