From 8d4de929af45b061f7113472f1a95bc1c0f80f10 Mon Sep 17 00:00:00 2001 From: Mohammad Taqui Sayed <6297436+mohdsayed@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:07:16 +0530 Subject: [PATCH 01/17] Add support forum link to PS landing page and dashboard (#848) * Add icons and link for support forum * Remove support forum item from sidebar * Add component for support link * Show support forum link conditionally * Handle dark mode --- .../src/components/landingPage/index.tsx | 34 ++++++++----- .../components/landingPage/supportLink.tsx | 48 +++++++++++++++++++ .../src/icons/external-link-black.svg | 4 +- packages/design-system/src/icons/index.tsx | 1 + packages/design-system/src/icons/support.svg | 3 ++ .../devtools/components/dashboard/index.tsx | 1 + .../components/privacySandbox/index.tsx | 6 ++- 7 files changed, 82 insertions(+), 15 deletions(-) create mode 100644 packages/design-system/src/components/landingPage/supportLink.tsx create mode 100644 packages/design-system/src/icons/support.svg diff --git a/packages/design-system/src/components/landingPage/index.tsx b/packages/design-system/src/components/landingPage/index.tsx index 6a76a646e..688012c16 100644 --- a/packages/design-system/src/components/landingPage/index.tsx +++ b/packages/design-system/src/components/landingPage/index.tsx @@ -25,6 +25,7 @@ import classNames from 'classnames'; */ import { ArrowUp } from '../../icons'; import ProgressBar from '../progressBar'; +import SupportLink from './supportLink'; import QuickLinksList from './quickLinksList'; import { PSInfoKeyType } from './infoCard/fetchPSInfo'; import InfoCard from './infoCard'; @@ -40,6 +41,7 @@ interface LandingPageProps { iframeBorderClass?: string; extraClasses?: string; showQuickLinks?: boolean; + showSupportLink?: boolean; } const LandingPage = ({ @@ -51,6 +53,7 @@ const LandingPage = ({ extraClasses, contentPanel, showQuickLinks = true, + showSupportLink = false, }: LandingPageProps) => { const [loading, setLoading] = useState(iframeSrc ? true : false); const [open, setOpen] = useState(true); @@ -67,19 +70,24 @@ const LandingPage = ({ 'divide-y divide-hex-gray dark:divide-quartz' )} > -
- - +
+
+ + +
+
+ {showSupportLink && } +
{ + return ( + + + + + Support Forum + + + + + ); +}; + +export default SupportLink; diff --git a/packages/design-system/src/icons/external-link-black.svg b/packages/design-system/src/icons/external-link-black.svg index 43209f757..7a598935d 100644 --- a/packages/design-system/src/icons/external-link-black.svg +++ b/packages/design-system/src/icons/external-link-black.svg @@ -1 +1,3 @@ - \ No newline at end of file + + + \ No newline at end of file diff --git a/packages/design-system/src/icons/index.tsx b/packages/design-system/src/icons/index.tsx index 2f4ba9b83..f39d80fad 100644 --- a/packages/design-system/src/icons/index.tsx +++ b/packages/design-system/src/icons/index.tsx @@ -82,3 +82,4 @@ export { default as ExternalLinkIcon } from './external-link.svg'; export { default as DoubleArrowIcon } from './double-arrow-icon.svg'; export { default as Settings } from './settings.svg'; export { default as MenuOpenIcon } from './menu-open-icon.svg'; +export { default as SupportIcon } from './support.svg'; diff --git a/packages/design-system/src/icons/support.svg b/packages/design-system/src/icons/support.svg new file mode 100644 index 000000000..54f2c7bbc --- /dev/null +++ b/packages/design-system/src/icons/support.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/packages/extension/src/view/devtools/components/dashboard/index.tsx b/packages/extension/src/view/devtools/components/dashboard/index.tsx index 17ac6819c..84348eadd 100644 --- a/packages/extension/src/view/devtools/components/dashboard/index.tsx +++ b/packages/extension/src/view/devtools/components/dashboard/index.tsx @@ -28,6 +28,7 @@ const Dashboard = () => { return ( } showQuickLinks={false} /> diff --git a/packages/extension/src/view/devtools/components/privacySandbox/index.tsx b/packages/extension/src/view/devtools/components/privacySandbox/index.tsx index dbbe4f405..1034de33d 100644 --- a/packages/extension/src/view/devtools/components/privacySandbox/index.tsx +++ b/packages/extension/src/view/devtools/components/privacySandbox/index.tsx @@ -34,7 +34,11 @@ const PrivacySandbox = () => { }, []); return ( - } /> + } + /> ); }; From 73e66e931d611ed7a082ab5f15f0f1d61f047b4f Mon Sep 17 00:00:00 2001 From: Kudaligi Amoghavarsha Date: Tue, 15 Oct 2024 10:19:43 +0530 Subject: [PATCH 02/17] Feature: Include individual site summaries in the aggregated report of a sitemap (#849) * Refactor function to generate new CSV * Fix heading for csv. * Fix header. * Move aggregated result to the bottom. Correct headings. --- .../generateRootSummaryDataCSV.ts | 94 ++++++++++++------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/packages/common/src/utils/generateReports/generateRootSummaryDataCSV.ts b/packages/common/src/utils/generateReports/generateRootSummaryDataCSV.ts index 28228b2bb..f7b13be2d 100644 --- a/packages/common/src/utils/generateReports/generateRootSummaryDataCSV.ts +++ b/packages/common/src/utils/generateReports/generateRootSummaryDataCSV.ts @@ -13,24 +13,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -/** - * External dependencies. - */ -import { I18n } from '@google-psat/i18n'; /** * Internal dependencies */ import type { CompleteJson, CookieTableData } from '../../cookies.types'; import extractReportData from '../extractReportData'; import reshapeCookies from '../reshapeCookies'; +import extractCookies from '../extractCookies'; -const generateRootSummaryDataCSV = ( - siteMapAnalysisData: CompleteJson[] -): string => { - const extractedData: { [key: string]: CookieTableData } = reshapeCookies( - extractReportData(siteMapAnalysisData).landingPageCookies - ); - +const calculateCSVData = (cookies: { + [key: string]: CookieTableData; +}): string => { let totalFirstPartyCookies = 0; let totalThirdPartyCookies = 0; let analyticsCookies = 0; @@ -45,8 +38,8 @@ const generateRootSummaryDataCSV = ( let totalCookies = 0; // eslint-disable-next-line complexity - Object.keys(extractedData).forEach((cookieKey) => { - const cookie = extractedData[cookieKey]; + Object.keys(cookies).forEach((cookieKey) => { + const cookie = cookies[cookieKey]; if (!cookie.analytics) { return; @@ -107,27 +100,62 @@ const generateRootSummaryDataCSV = ( totalCookies += 1; }); - const summary = { - [I18n.getMessage('totalCookies')]: totalCookies, - [I18n.getMessage('totalFirstPartyCookies')]: totalFirstPartyCookies, - [I18n.getMessage('totalThirdPartyCookies')]: totalThirdPartyCookies, - [I18n.getMessage('analyticsCookies')]: analyticsCookies, - [I18n.getMessage('functionalCookies')]: functionalCookies, - [I18n.getMessage('marketingCookies')]: marketingCookies, - [I18n.getMessage('uncategorizedCookies')]: uncategorisedCookies, - [I18n.getMessage('cookiesWithIssues')]: cookiesWithIssues, - [I18n.getMessage('analyticsCookiesWithIssues')]: analyticsCookiesWithIssues, - [I18n.getMessage('functionalCookiesWithIssues')]: - functionalCookiesWithIssues, - [I18n.getMessage('marketingCookiesWithIssues')]: marketingCookiesWithIssues, - [I18n.getMessage('uncategorizedCookiesWithIssues')]: - uncategorisedCookiesWithIssues, - }; + const summary = [ + totalCookies, + totalFirstPartyCookies, + totalThirdPartyCookies, + analyticsCookies, + functionalCookies, + marketingCookies, + uncategorisedCookies, + cookiesWithIssues, + analyticsCookiesWithIssues, + functionalCookiesWithIssues, + marketingCookiesWithIssues, + uncategorisedCookiesWithIssues, + ]; + + return summary.join(',').concat('\r\n'); +}; +export const generateRootSummaryDataCSV = ( + siteMapAnalysisData: CompleteJson[] +) => { + const extractedData: { [key: string]: CookieTableData } = reshapeCookies( + extractReportData(siteMapAnalysisData).landingPageCookies + ); + const headers = [ + 'URL', + 'Total Cookies', + 'First Party Cookies', + 'Third Party Cookies', + 'Analytics Cookies', + 'Functional Cookies', + 'Marketing Cookies', + 'Uncategorised Cookies', + 'Total Cookies With Issues', + 'Analytics Cookies With Issues', + 'Functional Cookies With Issues', + 'Marketing Cookies With Issues', + 'Uncategorised Cookies With Issues', + ]; + + let csvData = headers.join(',').concat('\r\n'); + + siteMapAnalysisData.forEach((singleSiteData) => { + const urlCookies = reshapeCookies( + extractCookies(singleSiteData.cookieData, '', false) + ); + + csvData = csvData + .concat(`${singleSiteData.pageUrl},`) + .concat(calculateCSVData(urlCookies)); + }); + + csvData = csvData + .concat('Aggregated,') + .concat(calculateCSVData(extractedData)); - return Object.entries(summary) - .map(([key, value]) => `${key}, ${value}`) - .join('\r\n') - .concat('\r\n'); + return csvData; }; export default generateRootSummaryDataCSV; From 5a123af76af63f5cf0659166460d7d678ec63d11 Mon Sep 17 00:00:00 2001 From: Amoghavarsha Kudaligi Date: Thu, 5 Dec 2024 21:06:36 +0530 Subject: [PATCH 03/17] Feature: Add warning and collect details of URL errors in sitemap (#830) * adding server error check * Throw error properly without printing the whole stack trace. * Add failure cross when analysis fails. Continue analysing in sitemap if single url is invalid. * Add support to collect all erroredOutUrls. * Move getSiteReport to utils package. * Add Urls with issues to cli dashboard. * Add support for stackTrace in error collection. * Add URL with issue icon. * Add error stack trace properly. * Fix failing tests * Fix cli e2e * Fix cli e2e and update the icon size. * Test cli e2e test passing. * Remove rmsync. * Change the url for testing. * Remove debug. * Skip CLI E2E tests. * Change description to error description. * Add a console.log message to indicate failing urls. * Add method to generate error logs. * Prettify stack trace. Add error.log in the stack trace. * Add feedback for the urls. * Fix bugs. * Fix name or urls. * Fix condition. * Fix QA feedbacks. * Fix url in sidebar. * use url instead of _url * Use shorthand and use index. * Refactor code. * Move error logs in common and download errorlog when using -o. * Address minor code feedbacks. --------- Co-authored-by: Fellyph Cintra --- .../src/browserManagement/index.ts | 436 +++++++++++------- .../analyzeCookiesUrlsAndFetchResources.ts | 108 +++-- ...zeCookiesUrlsInBatchesAndFetchResources.ts | 102 ++-- packages/cli-dashboard/src/app.tsx | 9 + packages/cli/src/e2e-tests/index.ts | 10 +- packages/cli/src/index.ts | 87 ++-- packages/cli/src/utils/getSiteReport.ts | 102 ++++ packages/cli/src/utils/index.ts | 2 + packages/cli/src/utils/saveReports.ts | 6 + packages/cli/src/utils/saveResultAsHTML.ts | 11 + packages/common/src/cookies.types.ts | 15 + packages/common/src/index.ts | 1 + .../common/src/utils/extractReportData.ts | 26 +- .../common/src/utils/generateErrorLogs.ts | 50 ++ .../src/utils/removeAndAddNewSpinnerText.ts | 6 +- .../sidebar/useSidebar/constants.ts | 1 + .../src/components/table/useTable/types.ts | 4 +- .../components/siteMapReport/index.tsx | 4 + .../components/siteMapReport/layout.tsx | 22 +- .../components/siteMapReport/sidebarData.ts | 18 + .../siteMapReport/urlsWithIssues.tsx | 34 ++ .../components/urlsWithIssues/index.tsx | 152 ++++++ .../generateSiteMapReportandDownload.ts | 7 +- 23 files changed, 879 insertions(+), 334 deletions(-) create mode 100644 packages/cli/src/utils/getSiteReport.ts create mode 100644 packages/common/src/utils/generateErrorLogs.ts create mode 100644 packages/report/src/dashboard/components/siteMapReport/urlsWithIssues.tsx create mode 100644 packages/report/src/dashboard/components/urlsWithIssues/index.tsx diff --git a/packages/analysis-utils/src/browserManagement/index.ts b/packages/analysis-utils/src/browserManagement/index.ts index bb0c9de3e..059669ef3 100644 --- a/packages/analysis-utils/src/browserManagement/index.ts +++ b/packages/analysis-utils/src/browserManagement/index.ts @@ -24,6 +24,7 @@ import { type ScriptTagUnderCheck, type LibraryData, type LibraryMatchers, + type SingleURLError, resolveWithTimeout, delay, RESPONSE_EVENT, @@ -51,12 +52,14 @@ export class BrowserManagement { isHeadless: boolean; pageWaitTime: number; pages: Record; + erroredOutUrls: Record; pageFrames: Record>; pageResponses: Record>; pageRequests: Record>; pageResourcesMaps: Record>; shouldLogDebug: boolean; spinnies: Spinnies | undefined; + isSiteMap: boolean; indent = 0; selectors: Selectors | undefined; constructor( @@ -65,6 +68,7 @@ export class BrowserManagement { pageWaitTime: number, shouldLogDebug: boolean, indent: number, + isSiteMap: boolean, spinnies?: Spinnies, selectors?: Selectors ) { @@ -72,6 +76,7 @@ export class BrowserManagement { this.browser = null; this.isHeadless = isHeadless; this.pageWaitTime = pageWaitTime; + this.isSiteMap = isSiteMap; this.pages = {}; this.pageFrames = {}; this.pageResponses = {}; @@ -81,13 +86,15 @@ export class BrowserManagement { this.spinnies = spinnies; this.indent = indent; this.selectors = selectors; + this.erroredOutUrls = {}; } - debugLog(msg: any) { + debugLog(msg: string, shouldShowWarning?: boolean) { if (this.shouldLogDebug && this.spinnies) { this.spinnies.add(msg, { text: msg, - succeedColor: 'white', + succeedColor: shouldShowWarning ? 'yellowBright' : 'white', + spinnerColor: shouldShowWarning ? 'yellowBright' : 'white', status: 'non-spinnable', indent: this.indent, }); @@ -116,20 +123,16 @@ export class BrowserManagement { async clickOnButtonUsingCMPSelectors(page: Page): Promise { let clickedOnButton = false; - try { - await Promise.all( - CMP_SELECTORS.map(async (selector) => { - const buttonToClick = await page.$(selector); - if (buttonToClick) { - await buttonToClick.click(); - clickedOnButton = true; - } - }) - ); - return clickedOnButton; - } catch (error) { - return clickedOnButton; - } + await Promise.all( + CMP_SELECTORS.map(async (selector) => { + const buttonToClick = await page.$(selector); + if (buttonToClick) { + await buttonToClick.click(); + clickedOnButton = true; + } + }) + ); + return clickedOnButton; } async clickOnGDPRUsingTextSelectors( @@ -140,89 +143,94 @@ export class BrowserManagement { return false; } - try { - const result = await page.evaluate((args: string[]) => { - const bannerNodes: Element[] = Array.from( - (document.querySelector('body')?.childNodes || []) as Element[] - ) - ?.filter((node: Element) => node && node?.tagName === 'DIV') - ?.filter((node) => { - if (!node || !node?.textContent) { - return false; - } - const regex = - /\b(consent|policy|cookie policy|privacy policy|personalize|preferences|cookies)\b/; + const result = await page.evaluate((args: string[]) => { + const bannerNodes: Element[] = Array.from( + (document.querySelector('body')?.childNodes || []) as Element[] + ) + ?.filter((node: Element) => node && node?.tagName === 'DIV') + ?.filter((node) => { + if (!node || !node?.textContent) { + return false; + } + const regex = + /\b(consent|policy|cookie policy|privacy policy|personalize|preferences|cookies)\b/; - return regex.test(node.textContent.toLowerCase()); - }); + return regex.test(node.textContent.toLowerCase()); + }); - return bannerNodes?.some((node: Element) => { - const buttonNodes = Array.from(node?.getElementsByTagName('button')); + return bannerNodes?.some((node: Element) => { + const buttonNodes = Array.from(node?.getElementsByTagName('button')); - return buttonNodes?.some((cnode) => { - if (!cnode?.textContent) { - return false; - } + return buttonNodes?.some((cnode) => { + if (!cnode?.textContent) { + return false; + } - return args.some((text) => { - if (cnode?.textContent?.toLowerCase().includes(text)) { - cnode?.click(); - return true; - } + return args.some((text) => { + if (cnode?.textContent?.toLowerCase().includes(text)) { + cnode?.click(); + return true; + } - return false; - }); + return false; }); }); - }, textSelectors); + }); + }, textSelectors); - return result; - } catch (error) { - return false; - } + return result; } async clickOnAcceptBanner(url: string) { - const page = this.pages[url]; + try { + const page = this.pages[url]; - if (!page) { - throw new Error('No page with the provided id was found'); - } + if (!page) { + throw new Error('No page with the provided id was found'); + } - const didSelectorsFromUserWork = await this.useSelectorsToSelectGDPRBanner( - page - ); + const didSelectorsFromUserWork = + await this.useSelectorsToSelectGDPRBanner(page); - if (didSelectorsFromUserWork) { - this.debugLog('GDPR banner found and accepted'); - await delay(this.pageWaitTime / 2); - return; - } + if (didSelectorsFromUserWork) { + this.debugLog('GDPR banner found and accepted'); + await delay(this.pageWaitTime / 2); + return; + } - // Click using CSS selectors. - const clickedUsingCMPCSSSelectors = - await this.clickOnButtonUsingCMPSelectors(page); + // Click using CSS selectors. + const clickedUsingCMPCSSSelectors = + await this.clickOnButtonUsingCMPSelectors(page); - if (clickedUsingCMPCSSSelectors) { - this.debugLog('GDPR banner found and accepted'); - await delay(this.pageWaitTime / 2); - return; - } + if (clickedUsingCMPCSSSelectors) { + this.debugLog('GDPR banner found and accepted'); + await delay(this.pageWaitTime / 2); + return; + } - const buttonClicked = await this.clickOnGDPRUsingTextSelectors( - page, - CMP_TEXT_SELECTORS - ); + const buttonClicked = await this.clickOnGDPRUsingTextSelectors( + page, + CMP_TEXT_SELECTORS + ); - if (buttonClicked) { - this.debugLog('GDPR banner found and accepted'); + if (buttonClicked) { + this.debugLog('GDPR banner found and accepted'); + await delay(this.pageWaitTime / 2); + return; + } + + this.debugLog('GDPR banner could not be found'); await delay(this.pageWaitTime / 2); return; + } catch (error) { + if (error instanceof Error) { + this.pushErrors(url, { + errorMessage: error.message, + stackTrace: error?.stack ?? '', + errorName: error?.name, + }); + } } - - this.debugLog('GDPR banner could not be found'); - await delay(this.pageWaitTime / 2); - return; } async useSelectorsToSelectGDPRBanner(page: Page): Promise { @@ -232,60 +240,56 @@ export class BrowserManagement { return false; } - try { - await Promise.all( - this.selectors?.cssSelectors.map(async (selector) => { - const buttonToClick = await page.$(selector); - if (buttonToClick) { - clickedOnButton = true; - this.debugLog('GDPR banner found and accepted'); - await buttonToClick.click(); - } - }) - ); - - if (clickedOnButton) { - return clickedOnButton; - } - - clickedOnButton = await page.evaluate((xPaths: string[]) => { - const rootElement = document.querySelector('html'); - - if (!rootElement) { - return false; + await Promise.all( + this.selectors?.cssSelectors.map(async (selector) => { + const buttonToClick = await page.$(selector); + if (buttonToClick) { + clickedOnButton = true; + this.debugLog('GDPR banner found and accepted'); + await buttonToClick.click(); } + }) + ); - return xPaths.some((xPath) => { - const _acceptButton = document - .evaluate(xPath, rootElement) - .iterateNext(); + if (clickedOnButton) { + return clickedOnButton; + } - if (!_acceptButton) { - return false; - } + clickedOnButton = await page.evaluate((xPaths: string[]) => { + const rootElement = document.querySelector('html'); - if (_acceptButton instanceof HTMLElement) { - _acceptButton?.click(); - return true; - } + if (!rootElement) { + return false; + } + + return xPaths.some((xPath) => { + const _acceptButton = document + .evaluate(xPath, rootElement) + .iterateNext(); + if (!_acceptButton) { return false; - }); - }, this.selectors?.xPath); + } - if (clickedOnButton) { - return clickedOnButton; - } + if (_acceptButton instanceof HTMLElement) { + _acceptButton?.click(); + return true; + } - clickedOnButton = await this.clickOnGDPRUsingTextSelectors( - page, - this.selectors?.textSelectors - ); + return false; + }); + }, this.selectors?.xPath); - return clickedOnButton; - } catch (error) { + if (clickedOnButton) { return clickedOnButton; } + + clickedOnButton = await this.clickOnGDPRUsingTextSelectors( + page, + this.selectors?.textSelectors + ); + + return clickedOnButton; } async openPage(): Promise { @@ -309,6 +313,14 @@ export class BrowserManagement { return sitePage; } + pushErrors(url: string, objectToPushed: SingleURLError) { + if (!this.erroredOutUrls[url]) { + this.erroredOutUrls[url] = []; + } + + this.erroredOutUrls[url].push(objectToPushed); + } + async navigateToPage(url: string) { const page = this.pages[url]; @@ -319,13 +331,43 @@ export class BrowserManagement { this.debugLog(`Starting navigation to URL: ${url}`); try { - await page.goto(url, { timeout: 10000 }); + const response = await page.goto(url, { + timeout: 10000, + }); + + const SUCCESS_RESPONSE = 200; + + if (response && response.status() !== SUCCESS_RESPONSE) { + this.pushErrors(url, { + errorMessage: `Invalid server response: ${response.status()}`, + errorCode: `${response.status()}`, + errorName: `INVALID_SERVER_RESPONSE`, + }); + + this.debugLog(`Warning: Server error found in URL: ${url}`, true); + + if (!this.isSiteMap) { + throw new Error(`Invalid server response: ${response.status()}`); + } + } + this.debugLog(`Navigation completed to URL: ${url}`); } catch (error) { - this.debugLog( - `Navigation did not finish in 10 seconds moving on to scrolling` - ); - //ignore + if (error instanceof Error) { + this.pushErrors(url, { + errorMessage: error.message, + stackTrace: error?.stack ?? '', + errorName: error?.name, + }); + + if (error?.name === 'TimeoutError') { + this.debugLog( + `Navigation did not finish on URL ${url} in 10 seconds moving on to scrolling` + ); + } + + throw error; + } } } @@ -746,40 +788,58 @@ export class BrowserManagement { url: string, Libraries: LibraryMatchers[] ) { - const page = this.pages[url]; + try { + const page = this.pages[url]; - if (!page) { - throw new Error('No page with the provided ID was found'); - } + if (!page) { + throw new Error('No page with the provided ID was found'); + } - const domQueryMatches: LibraryData = {}; + const domQueryMatches: LibraryData = {}; - await Promise.all( - Libraries.map(async ({ domQueryFunction, name }) => { - if (domQueryFunction && name) { - await page.addScriptTag({ - content: `window.${name.replaceAll('-', '')} = ${domQueryFunction}`, - }); + await Promise.all( + Libraries.map(async ({ domQueryFunction, name }) => { + if (domQueryFunction && name) { + await page.addScriptTag({ + content: `window.${name.replaceAll( + '-', + '' + )} = ${domQueryFunction}`, + }); - const queryResult = await page.evaluate((library: string) => { - //@ts-ignore - const functionDOMQuery = window[`${library}`]; + const queryResult = await page.evaluate((library: string) => { + //@ts-ignore + const functionDOMQuery = window[`${library}`]; - if (!functionDOMQuery) { - return []; - } + if (!functionDOMQuery) { + return []; + } - return functionDOMQuery(); - }, name.replaceAll('-', '')); + return functionDOMQuery(); + }, name.replaceAll('-', '')); - domQueryMatches[name] = { - domQuerymatches: queryResult as [string], - }; - } - }) - ); - const mainFrameUrl = new URL(page.url()).origin; - return { [mainFrameUrl]: domQueryMatches }; + domQueryMatches[name] = { + domQuerymatches: queryResult as [string], + }; + } + }) + ); + + const mainFrameUrl = new URL(page.url()).origin; + + return { [mainFrameUrl]: domQueryMatches }; + } catch (error) { + if (error instanceof Error) { + this.pushErrors(url, { + errorMessage: error.message, + stackTrace: error?.stack ?? '', + errorName: error?.name, + }); + + throw error; + } + return {}; + } } async analyzeCookies( @@ -798,11 +858,18 @@ export class BrowserManagement { ); // Navigate to URLs - await Promise.all( - userProvidedUrls.map(async (url) => { - await this.navigateToPage(url); - }) - ); + // eslint-disable-next-line no-useless-catch -- Because we are rethrowing the same error no need to create a new Error instance + try { + await Promise.all( + userProvidedUrls.map(async (url) => { + await this.navigateToPage(url); + }) + ); + } catch (error) { + if (!this.isSiteMap) { + throw error; + } + } // Delay for page to load resources await delay(this.pageWaitTime / 2); @@ -810,12 +877,17 @@ export class BrowserManagement { // Accept Banners if (!shouldSkipAcceptBanner) { // delay - - await Promise.all( - userProvidedUrls.map(async (url) => { - await this.clickOnAcceptBanner(url); - }) - ); + try { + await Promise.all( + userProvidedUrls.map(async (url) => { + await this.clickOnAcceptBanner(url); + }) + ); + } catch (error) { + if (!this.isSiteMap) { + throw error; + } + } } // Scroll to bottom of the page @@ -825,19 +897,25 @@ export class BrowserManagement { }) ); - await Promise.all( - userProvidedUrls.map(async (url) => { - const newMatches = await this.insertAndRunDOMQueryFunctions( - url, - Libraries - ); + try { + await Promise.all( + userProvidedUrls.map(async (url) => { + const newMatches = await this.insertAndRunDOMQueryFunctions( + url, + Libraries + ); - consolidatedDOMQueryMatches = { - ...consolidatedDOMQueryMatches, - ...newMatches, - }; - }) - ); + consolidatedDOMQueryMatches = { + ...consolidatedDOMQueryMatches, + ...newMatches, + }; + }) + ); + } catch (error) { + if (!this.isSiteMap) { + throw error; + } + } // Delay for page to load more resources await delay(this.pageWaitTime / 2); @@ -892,7 +970,11 @@ export class BrowserManagement { }) ); - return { result, consolidatedDOMQueryMatches }; + return { + result, + consolidatedDOMQueryMatches, + erroredOutUrls: this.erroredOutUrls, + }; } async deinitialize() { diff --git a/packages/analysis-utils/src/procedures/analyzeCookiesUrlsAndFetchResources.ts b/packages/analysis-utils/src/procedures/analyzeCookiesUrlsAndFetchResources.ts index 4865ee619..4e4e39537 100644 --- a/packages/analysis-utils/src/procedures/analyzeCookiesUrlsAndFetchResources.ts +++ b/packages/analysis-utils/src/procedures/analyzeCookiesUrlsAndFetchResources.ts @@ -39,65 +39,77 @@ export const analyzeCookiesUrlsAndFetchResources = async ( cookieDictionary: CookieDatabase, shouldSkipAcceptBanner: boolean, verbose: boolean, + isSitemap: boolean, selectors?: Selectors, spinnies?: Spinnies, indent = 4 ) => { - const browser = new BrowserManagement( - { - width: 1440, - height: 790, - deviceScaleFactor: 1, - }, - isHeadless, - delayTime, - verbose, - indent, - spinnies, - selectors - ); + // eslint-disable-next-line no-useless-catch -- Because we are rethrowing the same error no need to create a new Error instance + try { + const browser = new BrowserManagement( + { + width: 1440, + height: 790, + deviceScaleFactor: 1, + }, + isHeadless, + delayTime, + verbose, + indent, + isSitemap, + spinnies, + selectors + ); - await browser.initializeBrowser(true); - const { result: analysisCookieData, consolidatedDOMQueryMatches } = - await browser.analyzeCookies(urls, shouldSkipAcceptBanner, Libraries); + await browser.initializeBrowser(true); - const resources = browser.getResources(urls); + const { + result: analysisCookieData, + consolidatedDOMQueryMatches, + erroredOutUrls, + } = await browser.analyzeCookies(urls, shouldSkipAcceptBanner, Libraries); - const res = analysisCookieData.map(({ url: pageUrl, cookieData }) => { - Object.entries(cookieData).forEach(([, frameData]) => { - const frameCookies = frameData.frameCookies; - Object.entries(frameCookies).forEach(([key, cookie]) => { - const analytics = findAnalyticsMatch( - cookie.parsedCookie.name, - cookieDictionary - ); + const resources = browser.getResources(urls); - frameCookies[key.trim()].analytics = { - platform: analytics?.platform || 'Unknown', - category: analytics?.category || 'Uncategorized', - gdprUrl: analytics?.gdprUrl || '', - description: analytics?.description, - }; + const res = analysisCookieData.map(({ url: pageUrl, cookieData }) => { + Object.entries(cookieData).forEach(([, frameData]) => { + const frameCookies = frameData.frameCookies; + Object.entries(frameCookies).forEach(([key, cookie]) => { + const analytics = findAnalyticsMatch( + cookie.parsedCookie.name, + cookieDictionary + ); - frameCookies[key.trim()].isFirstParty = isFirstParty( - cookie.parsedCookie.domain, - pageUrl - ); + frameCookies[key.trim()].analytics = { + platform: analytics?.platform || 'Unknown', + category: analytics?.category || 'Uncategorized', + gdprUrl: analytics?.gdprUrl || '', + description: analytics?.description, + }; - frameCookies[key.trim()].blockingStatus = deriveBlockingStatus( - cookie.networkEvents - ); + frameCookies[key.trim()].isFirstParty = isFirstParty( + cookie.parsedCookie.domain, + pageUrl + ); + + frameCookies[key.trim()].blockingStatus = deriveBlockingStatus( + cookie.networkEvents + ); + }); }); - }); - return { - url: pageUrl, - cookieData, - resources: resources[pageUrl], - domQueryMatches: consolidatedDOMQueryMatches[pageUrl], - }; - }); + return { + url: pageUrl, + cookieData, + resources: resources[pageUrl], + erroredOutUrls, + domQueryMatches: consolidatedDOMQueryMatches[pageUrl], + }; + }); - await browser.deinitialize(); - return res; + await browser.deinitialize(); + return res; + } catch (error) { + throw error; + } }; diff --git a/packages/analysis-utils/src/procedures/analyzeCookiesUrlsInBatchesAndFetchResources.ts b/packages/analysis-utils/src/procedures/analyzeCookiesUrlsInBatchesAndFetchResources.ts index 4c5d3286e..382f46177 100644 --- a/packages/analysis-utils/src/procedures/analyzeCookiesUrlsInBatchesAndFetchResources.ts +++ b/packages/analysis-utils/src/procedures/analyzeCookiesUrlsInBatchesAndFetchResources.ts @@ -24,6 +24,7 @@ import { type LibraryMatchers, removeAndAddNewSpinnerText, type Selectors, + type SingleURLError, } from '@google-psat/common'; /** @@ -44,60 +45,67 @@ export const analyzeCookiesUrlsInBatchesAndFetchResources = async ( indent = 4, selectors?: Selectors ) => { - let report: { - url: string; - cookieData: { - [frameUrl: string]: { - frameCookies: { - [key: string]: CookieData; + // eslint-disable-next-line no-useless-catch -- Because we are rethrowing the same error no need to create a new Error instance + try { + let report: { + url: string; + cookieData: { + [frameUrl: string]: { + frameCookies: { + [key: string]: CookieData; + }; }; }; - }; - resources: { - origin: string | null; - content: string; - type?: string; - }[]; - domQueryMatches: LibraryData; - }[] = []; + resources: { + origin: string | null; + content: string; + type?: string; + }[]; + domQueryMatches: LibraryData; + erroredOutUrls: Record; + }[] = []; - for (let i = 0; i < urls.length; i += batchSize) { - const start = i; - const end = Math.min(urls.length - 1, i + batchSize - 1); + for (let i = 0; i < urls.length; i += batchSize) { + const start = i; + const end = Math.min(urls.length - 1, i + batchSize - 1); - spinnies && - indent === 4 && - spinnies.add(`cookie-batch-spinner${start + 1}-${end + 1}`, { - text: `Analyzing cookies in URLs ${start + 1} - ${end + 1}`, - indent, - }); + spinnies && + indent === 4 && + spinnies.add(`cookie-batch-spinner${start + 1}-${end + 1}`, { + text: `Analyzing cookies in URLs ${start + 1} - ${end + 1}`, + indent, + }); - const urlsWindow = urls.slice(start, end + 1); + const urlsWindow = urls.slice(start, end + 1); - const cookieAnalysisAndFetchedResources = - await analyzeCookiesUrlsAndFetchResources( - urlsWindow, - Libraries, - isHeadless, - delayTime, - cookieDictionary, - shouldSkipAcceptBanner, - verbose, - selectors, - spinnies - ); + const cookieAnalysisAndFetchedResources = + await analyzeCookiesUrlsAndFetchResources( + urlsWindow, + Libraries, + isHeadless, + delayTime, + cookieDictionary, + shouldSkipAcceptBanner, + verbose, + urls.length > 1, + selectors, + spinnies + ); - report = [...report, ...cookieAnalysisAndFetchedResources]; + report = [...report, ...cookieAnalysisAndFetchedResources]; - spinnies && - indent === 4 && - removeAndAddNewSpinnerText( - spinnies, - `cookie-batch-spinner${start + 1}-${end + 1}`, - `Done analyzing cookies in URLs ${start + 1} - ${end + 1}`, - indent - ); - } + spinnies && + indent === 4 && + removeAndAddNewSpinnerText( + spinnies, + `cookie-batch-spinner${start + 1}-${end + 1}`, + `Done analyzing cookies in URLs ${start + 1} - ${end + 1}`, + indent + ); + } - return report; + return report; + } catch (error) { + throw error; + } }; diff --git a/packages/cli-dashboard/src/app.tsx b/packages/cli-dashboard/src/app.tsx index cef985265..b70578e28 100644 --- a/packages/cli-dashboard/src/app.tsx +++ b/packages/cli-dashboard/src/app.tsx @@ -21,6 +21,7 @@ import { type CompleteJson, type CookieFrameStorageType, type LibraryData, + type ErroredOutUrlsData, extractReportData, extractCookies, } from '@google-psat/common'; @@ -39,11 +40,17 @@ enum DisplayType { const App = () => { const [cookies, setCookies] = useState({}); + const [landingPageCookies, setLandingPageCookies] = useState({}); const [completeJsonReport, setCompleteJsonReport] = useState< CompleteJson[] | null >(null); + + const [erroredOutUrls, setErroredOutUrls] = useState( + [] + ); + const [libraryMatches, setLibraryMatches] = useState<{ [key: string]: LibraryData; } | null>(null); @@ -104,6 +111,7 @@ const App = () => { _libraryMatches = extractedData.consolidatedLibraryMatches; setLandingPageCookies(extractedData.landingPageCookies); + setErroredOutUrls(extractedData.erroredOutUrlsData); } else { _cookies = extractCookies(data[0].cookieData, '', true); _libraryMatches = { [data[0].pageUrl]: data[0].libraryMatches }; @@ -116,6 +124,7 @@ const App = () => { if (type === DisplayType.SITEMAP) { return ( { +xdescribe('CLI E2E Test', () => { const cli = require.resolve('../../dist/main.js'); - afterAll(() => { - fs.rmSync(path.join(process.cwd(), '/out/bbc-com'), { recursive: true }); - }); - it('Should run site analysis', () => { return coffee - .fork(cli, ['-u https://bbc.com', '-w 100']) + .fork(cli, ['-u https://bbc.com', '-w 1000']) .includes('stdout', '/out/bbc-com/report_') .end(); }, 60000); diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index c3e810344..bce77a24a 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -24,17 +24,9 @@ import { existsSync } from 'fs-extra'; import Spinnies from 'spinnies'; import path, { basename } from 'path'; import { I18n } from '@google-psat/i18n'; -import { - type CompleteJson, - type LibraryData, - removeAndAddNewSpinnerText, -} from '@google-psat/common'; +import { removeAndAddNewSpinnerText } from '@google-psat/common'; import { analyzeCookiesUrlsInBatchesAndFetchResources } from '@google-psat/analysis-utils'; -import { - DetectionFunctions, - LIBRARIES, - detectMatchingSignatures, -} from '@google-psat/library-detection'; +import { LIBRARIES } from '@google-psat/library-detection'; import { pathToFileURL } from 'node:url'; /** @@ -51,12 +43,13 @@ import { filePathValidator, urlValidator, numericValidator, + redLogger, + getSiteReport, + saveResultsAsHTML, } from './utils'; -import { redLogger } from './utils/coloredLoggers'; -import saveResultsAsHTML from './utils/saveResultAsHTML'; -import getSelectorsFromPath from './utils/getSelectorsFromPath'; -import checkLatestVersion from './utils/checkLatestVersion'; import packageJson from '../package.json'; +import checkLatestVersion from './utils/checkLatestVersion'; +import getSelectorsFromPath from './utils/getSelectorsFromPath'; events.EventEmitter.defaultMaxListeners = 15; @@ -243,44 +236,46 @@ program.parse(); text: 'Analyzing cookies on the first site visit', }); - const cookieAnalysisAndFetchedResourceData = - await analyzeCookiesUrlsInBatchesAndFetchResources( - urlsToProcess, - LIBRARIES, - !isHeadful, - waitTime, - cookieDictionary, - concurrency, - spinnies, - shouldSkipAcceptBanner, - verbose, - sitemapUrl || filePath ? 4 : 3, - selectors - ); - + let cookieAnalysisAndFetchedResourceData: any; + + // eslint-disable-next-line no-useless-catch -- Because we are rethrowing the same error no need to create a new Error instance + try { + cookieAnalysisAndFetchedResourceData = + await analyzeCookiesUrlsInBatchesAndFetchResources( + urlsToProcess, + LIBRARIES, + !isHeadful, + waitTime, + cookieDictionary, + concurrency, + spinnies, + shouldSkipAcceptBanner, + verbose, + sitemapUrl || filePath ? 4 : 3, + selectors + ); + } catch (error) { + if (urlsToProcess.length === 1) { + removeAndAddNewSpinnerText( + spinnies, + 'cookie-spinner', + 'Failure in analyzing cookies!', + 0, + true + ); + throw error; + } + } removeAndAddNewSpinnerText( spinnies, 'cookie-spinner', 'Done analyzing cookies!' ); - const result = urlsToProcess.map((_url, ind) => { - const detectedMatchingSignatures: LibraryData = { - ...detectMatchingSignatures( - cookieAnalysisAndFetchedResourceData[ind].resources ?? [], - Object.fromEntries( - LIBRARIES.map((library) => [library.name, library.detectionFunction]) - ) as DetectionFunctions - ), - ...(cookieAnalysisAndFetchedResourceData[ind]?.domQueryMatches ?? {}), - }; - return { - pageUrl: _url, - psatVersion: packageJson.version, // For adding in downloaded JSON file. - cookieData: cookieAnalysisAndFetchedResourceData[ind].cookieData, - libraryMatches: detectedMatchingSignatures ?? [], - } as unknown as CompleteJson; - }); + const result = getSiteReport( + urlsToProcess, + cookieAnalysisAndFetchedResourceData + ); I18n.loadCLIMessagesData(locale); diff --git a/packages/cli/src/utils/getSiteReport.ts b/packages/cli/src/utils/getSiteReport.ts new file mode 100644 index 000000000..221c1dbfb --- /dev/null +++ b/packages/cli/src/utils/getSiteReport.ts @@ -0,0 +1,102 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * External dependencies + */ +import { + parseUrl, + type CompleteJson, + type ErroredOutUrlsData, + type SingleURLError, +} from '@google-psat/common'; +import { + type DetectionFunctions, + detectMatchingSignatures, + LIBRARIES, + type LibraryData, +} from '@google-psat/library-detection'; + +/** + * This function returns the exact object which will be sent to the cli dashboard when report is generated. + * @param urls The user provided urls to be processed. + * @param processedData The cookie data along with library detection information with erroredOutUrls. + * @returns {object} The object which will be used to send the report to the cli dashboard. + */ +function getSiteReport(urls: string[], processedData: any) { + return urls.map((url, index) => { + const { + erroredOutUrls = {}, + cookieData = {}, + domQueryMatches = {}, + resources = [], + } = processedData[index]; + + const hasTimeOutError = erroredOutUrls[url]?.some( + ({ errorName }: SingleURLError) => + errorName === 'TimeoutError' || errorName === 'i' + ); + + const detectedMatchingSignatures: LibraryData = { + ...detectMatchingSignatures( + resources ?? [], + Object.fromEntries( + LIBRARIES.map((library) => [library.name, library.detectionFunction]) + ) as DetectionFunctions + ), + ...(domQueryMatches ?? {}), + }; + + if (erroredOutUrls[url] && erroredOutUrls[url].length > 0) { + if (hasTimeOutError) { + return { + pageUrl: parseUrl(url) ? new URL(url).href : encodeURI(url), + cookieData: cookieData, + libraryMatches: detectedMatchingSignatures ?? [], + erroredOutUrls: [ + ...erroredOutUrls[url].map((errors: SingleURLError) => { + return { + url: parseUrl(url) ? new URL(url).href : encodeURI(url), + ...errors, + }; + }), + ] as ErroredOutUrlsData[], + } as unknown as CompleteJson; + } + + return { + pageUrl: parseUrl(url) ? new URL(url).href : encodeURI(url), + cookieData: {}, + libraryMatches: [], + erroredOutUrls: [ + ...erroredOutUrls[url].map((errors: SingleURLError) => { + return { + url, + ...errors, + }; + }), + ] as ErroredOutUrlsData[], + } as unknown as CompleteJson; + } + + return { + pageUrl: parseUrl(url) ? new URL(url).href : encodeURI(url), + cookieData, + libraryMatches: detectedMatchingSignatures ?? [], + } as unknown as CompleteJson; + }); +} + +export default getSiteReport; diff --git a/packages/cli/src/utils/index.ts b/packages/cli/src/utils/index.ts index 7502f218b..7278c3a73 100644 --- a/packages/cli/src/utils/index.ts +++ b/packages/cli/src/utils/index.ts @@ -22,4 +22,6 @@ export { default as askUserInput } from './askUserInput'; export { default as generatePrefix } from './generatePrefix'; export { default as getOutputFilePath } from './getOutputFilePath'; export { default as saveResultsAsHTML } from './saveResultAsHTML'; +export { default as getSiteReport } from './getSiteReport'; +export * from './coloredLoggers'; export * from './validators'; diff --git a/packages/cli/src/utils/saveReports.ts b/packages/cli/src/utils/saveReports.ts index 94e73898c..2de54380d 100644 --- a/packages/cli/src/utils/saveReports.ts +++ b/packages/cli/src/utils/saveReports.ts @@ -18,6 +18,7 @@ * External dependencies. */ import { + generateErrorLogFile, generateRootSummaryDataCSV, type CompleteJson, } from '@google-psat/common'; @@ -62,6 +63,11 @@ const saveReports = async ( const rootSummaryData = generateRootSummaryDataCSV(result); await ensureFile(path.join(outDir, 'report.csv')); await writeFile(path.join(outDir, 'report.csv'), rootSummaryData); + + const errorLogs = generateErrorLogFile(result); + + await ensureFile(path.join(outDir, 'error_logs.txt')); + await writeFile(path.join(outDir, 'error_logs.txt'), errorLogs); // Sitemap report await Promise.all( result.map(async (siteReport) => { diff --git a/packages/cli/src/utils/saveResultAsHTML.ts b/packages/cli/src/utils/saveResultAsHTML.ts index 1415bd300..f431ec624 100644 --- a/packages/cli/src/utils/saveResultAsHTML.ts +++ b/packages/cli/src/utils/saveResultAsHTML.ts @@ -120,6 +120,17 @@ const saveResultsAsHTML = async ( writeFile(outputFilePath, buffer, () => { if (!fileName) { + if ( + result.some( + (singleResult) => + singleResult.erroredOutUrls && + singleResult.erroredOutUrls.length > 0 + ) + ) { + console.log( + `\nWarning: Some URLs encountered issues while analysing cookies. Please check the dashboard for more details.` + ); + } console.log(`\nReport: ${URL.pathToFileURL(outFileFullDir)}`); } }); diff --git a/packages/common/src/cookies.types.ts b/packages/common/src/cookies.types.ts index f1cc8e9a8..5b218f745 100644 --- a/packages/common/src/cookies.types.ts +++ b/packages/common/src/cookies.types.ts @@ -131,6 +131,13 @@ export type CookieTableData = CookieData & { isDomainInAllowList?: boolean; }; +export type ErroredOutUrlsData = { + errorCode?: string; + errorMessage: string; + url: string; + stackTrace?: string; + errorName: string; +}; export interface TabCookies { [key: string]: CookieTableData; } @@ -217,6 +224,13 @@ export type CookieFrameStorageType = { }; }; +export type SingleURLError = { + errorMessage: string; + errorCode?: string; + stackTrace?: string; + errorName: string; +}; + export type CompleteJson = { pageUrl: string; cookieData: { @@ -227,6 +241,7 @@ export type CompleteJson = { frameType?: string | undefined; }; }; + erroredOutUrls: ErroredOutUrlsData[]; libraryMatches: { [key: string]: LibraryData }; }; diff --git a/packages/common/src/index.ts b/packages/common/src/index.ts index 77092ddc4..f4179154a 100644 --- a/packages/common/src/index.ts +++ b/packages/common/src/index.ts @@ -24,6 +24,7 @@ export { } from './utils/findAnalyticsMatch'; export { default as removeAndAddNewSpinnerText } from './utils/removeAndAddNewSpinnerText'; export { default as calculateEffectiveExpiryDate } from './utils/calculateEffectiveExpiryDate'; +export { default as generateErrorLogFile } from './utils/generateErrorLogs'; export { default as sanitizeCsvRecord } from './utils/sanitizeCsvRecord'; export { parseUrl } from './utils/parseUrl'; export { default as fetchLocalData } from './utils/fetchLocalData'; diff --git a/packages/common/src/utils/extractReportData.ts b/packages/common/src/utils/extractReportData.ts index 66a9e6f1f..00519ccc2 100644 --- a/packages/common/src/utils/extractReportData.ts +++ b/packages/common/src/utils/extractReportData.ts @@ -17,14 +17,37 @@ /** * Internal dependencies */ -import { CompleteJson, CookieFrameStorageType } from '../cookies.types'; +import { + CompleteJson, + CookieFrameStorageType, + ErroredOutUrlsData, +} from '../cookies.types'; import { LibraryData } from '../libraryDetection.types'; import extractCookies from './extractCookies'; const extractReportData = (data: CompleteJson[]) => { const landingPageCookies = {}; + const erroredOutUrlsData: ErroredOutUrlsData[] = []; const consolidatedLibraryMatches: { [url: string]: LibraryData } = {}; + data.forEach(({ cookieData, pageUrl, libraryMatches, erroredOutUrls }) => { + erroredOutUrlsData.push(...(erroredOutUrls ?? [])); + + if ( + erroredOutUrls && + erroredOutUrls.filter(({ url }) => url === pageUrl).length > 0 + ) { + return; + } + + formatCookieData( + extractCookies(cookieData, pageUrl, true), + landingPageCookies + ); + + consolidatedLibraryMatches[pageUrl] = libraryMatches; + }); + data.forEach(({ cookieData, pageUrl, libraryMatches }) => { formatCookieData( extractCookies(cookieData, pageUrl, true), @@ -37,6 +60,7 @@ const extractReportData = (data: CompleteJson[]) => { return { landingPageCookies, consolidatedLibraryMatches, + erroredOutUrlsData, }; }; diff --git a/packages/common/src/utils/generateErrorLogs.ts b/packages/common/src/utils/generateErrorLogs.ts new file mode 100644 index 000000000..e14611d2a --- /dev/null +++ b/packages/common/src/utils/generateErrorLogs.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Internal dependencies + */ +import { CompleteJson } from '../cookies.types'; + +/** + * This function generates error logs file for a sitemap analysis. + * @param JSONReport The JSON report for which error logs file has to be generated + * @returns string of error logs. + */ +export const generateErrorLogFile = (JSONReport: CompleteJson[]) => { + let erroredOutTextFileData = ''; + + JSONReport.forEach(({ erroredOutUrls }) => { + if (!erroredOutUrls) { + return; + } + + erroredOutUrls.forEach((error) => { + const temporaryFormedData = ` + URL: ${error.url} + Error Code: ${error.errorCode ?? 'N/A'} + Error Message: ${error.errorMessage} + ErrorStack: + ${error.stackTrace ?? 'N/A'} + `; + + erroredOutTextFileData += temporaryFormedData + '\n'; + }); + }); + + return erroredOutTextFileData; +}; + +export default generateErrorLogFile; diff --git a/packages/common/src/utils/removeAndAddNewSpinnerText.ts b/packages/common/src/utils/removeAndAddNewSpinnerText.ts index f02757a56..1152f3635 100644 --- a/packages/common/src/utils/removeAndAddNewSpinnerText.ts +++ b/packages/common/src/utils/removeAndAddNewSpinnerText.ts @@ -19,16 +19,18 @@ * @param spinnerName name of the spinner. * @param newSpinnerText The text to be added to the new spinner. * @param indent The indentation for the new text. + * @param failure This determines if updted spinner should show fail status. */ export default function removeAndAddNewSpinnerText( spinnies: any, spinnerName: string, newSpinnerText: string, - indent = 0 + indent = 0, + failure = false ) { spinnies.add(`${spinnerName}-succees`, { text: newSpinnerText, - status: 'succeed', + status: failure ? 'fail' : 'succeed', indent, }); diff --git a/packages/design-system/src/components/sidebar/useSidebar/constants.ts b/packages/design-system/src/components/sidebar/useSidebar/constants.ts index 4b1507c6e..03910d3f5 100644 --- a/packages/design-system/src/components/sidebar/useSidebar/constants.ts +++ b/packages/design-system/src/components/sidebar/useSidebar/constants.ts @@ -18,6 +18,7 @@ export enum SIDEBAR_ITEMS_KEYS { DASHBOARD = 'dashboard', COOKIES = 'cookies', COOKIES_WITH_ISSUES = 'cookie-issues', + URL_WITH_ISSUES = 'urls-with-issues', PRIVACY_SANDBOX = 'privacy-sandbox', SITE_BOUNDARIES = 'site-boundaries', CHIPS = 'chips', diff --git a/packages/design-system/src/components/table/useTable/types.ts b/packages/design-system/src/components/table/useTable/types.ts index 808bc326f..1fa5dcf98 100644 --- a/packages/design-system/src/components/table/useTable/types.ts +++ b/packages/design-system/src/components/table/useTable/types.ts @@ -16,9 +16,9 @@ /** * External dependencies. */ -import type { CookieTableData } from '@google-psat/common'; +import type { CookieTableData, ErroredOutUrlsData } from '@google-psat/common'; -export type TableData = CookieTableData & { +export type TableData = (CookieTableData | ErroredOutUrlsData) & { highlighted?: boolean; }; diff --git a/packages/report/src/dashboard/components/siteMapReport/index.tsx b/packages/report/src/dashboard/components/siteMapReport/index.tsx index 8628dca45..e2872dcd3 100644 --- a/packages/report/src/dashboard/components/siteMapReport/index.tsx +++ b/packages/report/src/dashboard/components/siteMapReport/index.tsx @@ -22,6 +22,7 @@ import type { CookieFrameStorageType, CompleteJson, LibraryData, + ErroredOutUrlsData, } from '@google-psat/common'; import { SidebarProvider, type SidebarItems } from '@google-psat/design-system'; @@ -36,6 +37,7 @@ interface SiteMapReportProps { completeJson: CompleteJson[] | null; path: string; libraryMatches: { [url: string]: LibraryData } | null; + erroredOutUrls: ErroredOutUrlsData[]; } const SiteMapReport = ({ @@ -43,12 +45,14 @@ const SiteMapReport = ({ completeJson, path, libraryMatches, + erroredOutUrls, }: SiteMapReportProps) => { const [data, setData] = useState(sidebarData); return ( >; path: string; libraryMatches: { [url: string]: LibraryData } | null; + erroredOutUrls: ErroredOutUrlsData[]; } const Layout = ({ @@ -57,14 +60,21 @@ const Layout = ({ sidebarData, setSidebarData, path, + erroredOutUrls, libraryMatches, }: LayoutProps) => { const [sites, setSites] = useState([]); useEffect(() => { const _sites = new Set(); - completeJson?.forEach(({ pageUrl }) => { - _sites.add(pageUrl); + completeJson?.forEach(({ pageUrl, erroredOutUrls: _erroredOutURLs }) => { + if ( + !_erroredOutURLs?.some( + ({ url, errorName }) => url === pageUrl && errorName !== 'i' + ) + ) { + _sites.add(pageUrl); + } }); setSites(Array.from(_sites)); @@ -185,9 +195,17 @@ const Layout = ({ }, }; + _data[SIDEBAR_ITEMS_KEYS.URL_WITH_ISSUES].panel = { + Element: ErroredOutUrls, + props: { + erroredOutUrls, + }, + }; + return _data; }); }, [ + erroredOutUrls, clearQuery, completeJson, cookiesWithIssues, diff --git a/packages/report/src/dashboard/components/siteMapReport/sidebarData.ts b/packages/report/src/dashboard/components/siteMapReport/sidebarData.ts index 4173ba334..5e60719d1 100644 --- a/packages/report/src/dashboard/components/siteMapReport/sidebarData.ts +++ b/packages/report/src/dashboard/components/siteMapReport/sidebarData.ts @@ -47,6 +47,24 @@ const sidebarData: SidebarItems = { }, }, }, + [SIDEBAR_ITEMS_KEYS.URL_WITH_ISSUES]: { + title: 'URL Issues', + children: {}, + icon: { + //@ts-ignore + Element: WarningBare, + props: { + className: 'fill-granite-gray', + }, + }, + selectedIcon: { + //@ts-ignore + Element: WarningBare, + props: { + className: 'fill-white', + }, + }, + }, }; export default sidebarData; diff --git a/packages/report/src/dashboard/components/siteMapReport/urlsWithIssues.tsx b/packages/report/src/dashboard/components/siteMapReport/urlsWithIssues.tsx new file mode 100644 index 000000000..61613d00f --- /dev/null +++ b/packages/report/src/dashboard/components/siteMapReport/urlsWithIssues.tsx @@ -0,0 +1,34 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * External dependencies. + */ +import type { ErroredOutUrlsData } from '@google-psat/common'; + +/** + * Internal dependencies. + */ +import ErroredOutUrls from '../urlsWithIssues'; + +interface URLSWithIssuesProps { + erroredOutUrls: ErroredOutUrlsData[]; +} + +const URLSWithIssues = ({ erroredOutUrls }: URLSWithIssuesProps) => { + return ; +}; + +export default URLSWithIssues; diff --git a/packages/report/src/dashboard/components/urlsWithIssues/index.tsx b/packages/report/src/dashboard/components/urlsWithIssues/index.tsx new file mode 100644 index 000000000..70e984abc --- /dev/null +++ b/packages/report/src/dashboard/components/urlsWithIssues/index.tsx @@ -0,0 +1,152 @@ +/* + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * External dependencies + */ +import { useMemo, useState } from 'react'; +import { Resizable } from 're-resizable'; +import { noop, type ErroredOutUrlsData } from '@google-psat/common'; +import { I18n } from '@google-psat/i18n'; +import { + Table, + TableProvider, + type TableColumn, + type InfoType, + type TableRow, + type TableFilter, +} from '@google-psat/design-system'; + +interface ErroredOutUrlsProps { + erroredOutUrls: ErroredOutUrlsData[]; +} + +const ErroredOutUrls = ({ erroredOutUrls }: ErroredOutUrlsProps) => { + const [selectedRow, setSelectedRow] = useState(); + + const tableColumns = useMemo( + () => [ + { + header: 'URL', + accessorKey: 'url', + cell: (info: InfoType) => info, + enableHiding: false, + }, + { + header: 'Error Description', + accessorKey: 'errorMessage', + cell: (info: InfoType) => info, + }, + { + header: 'Error Code', + accessorKey: 'errorCode', + cell: (info: InfoType) => ( + {info} + ), + }, + ], + [] + ); + + const filters = useMemo(() => ({}), []); + + return ( +
+ + { + setSelectedRow(row as ErroredOutUrlsData); + }} + onRowContextMenu={noop} + getRowObjectKey={(row: TableRow) => { + return (row.originalData as ErroredOutUrlsData).url; + }} + > + + + +
+ {selectedRow ? ( +
+ {selectedRow.url && ( + <> +

+ Error Message +

+

+ {selectedRow.errorMessage} +

+ + )} + <> +

+ Error code +

+

+ {selectedRow?.errorCode || I18n.getMessage('noDescription')} +

+ + {selectedRow?.stackTrace && ( + <> +

+ Stack trace +

+

+ 

+                    
+
+

+ + )} +
+ ) : ( +
+

+ {I18n.getMessage('selectRowToPreview')} +

+
+ )} +
+ + ); +}; + +export default ErroredOutUrls; diff --git a/packages/report/src/dashboard/components/utils/reportDownloader/generateSiteMapReportandDownload.ts b/packages/report/src/dashboard/components/utils/reportDownloader/generateSiteMapReportandDownload.ts index bc2375f08..4216addb5 100644 --- a/packages/report/src/dashboard/components/utils/reportDownloader/generateSiteMapReportandDownload.ts +++ b/packages/report/src/dashboard/components/utils/reportDownloader/generateSiteMapReportandDownload.ts @@ -20,9 +20,10 @@ import JSZip from 'jszip'; import { saveAs } from 'file-saver'; import { + type CompleteJson, generateRootSummaryDataCSV, getCurrentDateAndTime, - type CompleteJson, + generateErrorLogFile, } from '@google-psat/common'; import { type TableFilter } from '@google-psat/design-system'; /** @@ -61,6 +62,10 @@ const generateSiteMapReportandDownload = async ( zip.file('report.html', report); zip.file('report.csv', rootSummaryData); + const errorLogs = generateErrorLogFile(JSONReport); + + zip.file('error_logs.txt', errorLogs); + const content = await zip.generateAsync({ type: 'blob' }); saveAs( content, From 86ef7e2c45b23d3c6ee8bf9aa0f033c07dac3ae9 Mon Sep 17 00:00:00 2001 From: Amoghavarsha Kudaligi Date: Thu, 5 Dec 2024 21:09:11 +0530 Subject: [PATCH 04/17] Fix linter issues. --- packages/common/src/test-utils/data.mock.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/common/src/test-utils/data.mock.ts b/packages/common/src/test-utils/data.mock.ts index 5c19327d4..fecff505e 100644 --- a/packages/common/src/test-utils/data.mock.ts +++ b/packages/common/src/test-utils/data.mock.ts @@ -18,6 +18,7 @@ import { CompleteJson } from '../cookies.types'; export const mockData1: CompleteJson = { pageUrl: 'https://edition.cnn.com/', + erroredOutUrls: [], cookieData: { 'https://edition.cnn.com': { frameCookies: { @@ -100,6 +101,7 @@ export const mockData1: CompleteJson = { export const mockData2: CompleteJson = { pageUrl: 'https://edition.cnn.com/', + erroredOutUrls: [], cookieData: { 'https://edition.cnn.com': { frameCookies: { @@ -227,6 +229,7 @@ export const mockData2: CompleteJson = { export const tempSinglePageData: CompleteJson = { pageUrl: 'https://edition.cnn.com/sitemaps/sitemap-section.xml', libraryMatches: {}, + erroredOutUrls: [], cookieData: { 'https://edition.cnn.com': { frameCookies: { @@ -260,6 +263,7 @@ export const tempSinglePageData: CompleteJson = { export const tempMultiPageData: CompleteJson[] = [ { libraryMatches: {}, + erroredOutUrls: [], pageUrl: 'https://www.cnn.com/index.html', cookieData: { 'https://edition.cnn.com': { @@ -293,6 +297,7 @@ export const tempMultiPageData: CompleteJson[] = [ }, { libraryMatches: {}, + erroredOutUrls: [], pageUrl: 'https://edition.cnn.com/index.html', cookieData: { 'https://edition.cnn.com': { From 59767cdea9c8e91a35a32552ac892b383fe46056 Mon Sep 17 00:00:00 2001 From: Amoghavarsha Kudaligi Date: Thu, 5 Dec 2024 21:28:11 +0530 Subject: [PATCH 05/17] Fix condition. --- packages/analysis-utils/src/browserManagement/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/analysis-utils/src/browserManagement/index.ts b/packages/analysis-utils/src/browserManagement/index.ts index 059669ef3..198711633 100644 --- a/packages/analysis-utils/src/browserManagement/index.ts +++ b/packages/analysis-utils/src/browserManagement/index.ts @@ -359,8 +359,8 @@ export class BrowserManagement { stackTrace: error?.stack ?? '', errorName: error?.name, }); - - if (error?.name === 'TimeoutError') { + console.log(JSON.stringify(error)); + if (error?.name === 'TimeoutError' || error?.name === 'i') { this.debugLog( `Navigation did not finish on URL ${url} in 10 seconds moving on to scrolling` ); From 1a891fa9c142185fe8f65e1434225c53756928d9 Mon Sep 17 00:00:00 2001 From: sayedtaqui Date: Fri, 29 Nov 2024 12:29:58 +0530 Subject: [PATCH 06/17] Replace support icon and update button text --- .../design-system/src/components/landingPage/supportLink.tsx | 5 ++++- packages/design-system/src/icons/support.svg | 4 +--- .../view/devtools/components/privacySandbox/contentPanel.tsx | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/design-system/src/components/landingPage/supportLink.tsx b/packages/design-system/src/components/landingPage/supportLink.tsx index a41ded8b0..c5ea47adb 100644 --- a/packages/design-system/src/components/landingPage/supportLink.tsx +++ b/packages/design-system/src/components/landingPage/supportLink.tsx @@ -32,7 +32,10 @@ const SupportLink = () => { rel="noreferrer" > - + Support Forum diff --git a/packages/design-system/src/icons/support.svg b/packages/design-system/src/icons/support.svg index 54f2c7bbc..f456b4e40 100644 --- a/packages/design-system/src/icons/support.svg +++ b/packages/design-system/src/icons/support.svg @@ -1,3 +1 @@ - - - \ No newline at end of file + \ No newline at end of file diff --git a/packages/extension/src/view/devtools/components/privacySandbox/contentPanel.tsx b/packages/extension/src/view/devtools/components/privacySandbox/contentPanel.tsx index 9b5bdb3ca..bd3260484 100644 --- a/packages/extension/src/view/devtools/components/privacySandbox/contentPanel.tsx +++ b/packages/extension/src/view/devtools/components/privacySandbox/contentPanel.tsx @@ -56,7 +56,7 @@ const ContentPanel = () => { target="__blank" className="bg-cultured-grey text-raisin-black py-2 px-9 rounded border border-dark-grey text-base hover:bg-light-gray hover:border-american-silver flex" > - Learn About Privacy Sandbox + Learn More