feat(route/nytimes): enhance full text fetch

DIYgod · Oct 24, 2024 · 3715992 · 3715992
1 parent 8aeb577
commit 3715992
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 7 deletions.
diff --git a/lib/routes/nytimes/index.ts b/lib/routes/nytimes/index.ts
@@ -81,7 +81,7 @@ async function handler(ctx) {
     const browser = await puppeteer();
     const feed = await parser.parseURL(rssUrl);
     const items = await Promise.all(
-        feed.items.splice(0, 10).map(async (item) => {
+        feed.items.splice(0, 3).map(async (item) => {
             let link = item.link;
 
             let response,
@@ -118,8 +118,9 @@ async function handler(ctx) {
                     if ($('.dual-btn').length > 0) {
                         hasEnVersion = true;
                         link = $('.dual-btn a').last().attr().href;
-
-                        response = await utils.PuppeterGetter(ctx, browser, link);
+                        if (link !== '') {
+                            response = await utils.PuppeterGetter(ctx, browser, link);
+                        }
                     }
                 }
             }

diff --git a/lib/routes/nytimes/utils.ts b/lib/routes/nytimes/utils.ts
@@ -19,11 +19,15 @@ const PuppeterGetter = async (ctx, browser, link) => {
         const page = await browser.newPage();
         await page.setRequestInterception(true);
         page.on('request', (request) => {
-            request.resourceType() === 'document' ? request.continue() : request.abort();
-        });
-        await page.goto(link, {
-            waitUntil: 'domcontentloaded',
+            if (request.url().includes('https://www.nytimes.com/svc/onsite-messaging/query') || request.url().includes('https://meter-svc.nytimes.com/meter.js')) {
+                request.abort();
+                return;
+            }
+            request.continue();
+            // request.resourceType() === 'document' ? request.continue() : request.abort();
         });
+        await page.goto(link);
+        await page.waitForSelector('[data-testid=optimistic-truncator-message]', { hidden: true, timeout: 0 });
         const response = await page.evaluate(() => document.querySelector('body').innerHTML);
         return response;
     });