From f272ce1cacf6ff36f174b63b486e91ac87c06b91 Mon Sep 17 00:00:00 2001 From: asong Date: Thu, 17 Oct 2024 12:07:06 +0800 Subject: [PATCH 01/18] add routes guozaoke --- lib/config.ts | 6 ++ lib/routes/guozaoke/index.ts | 147 +++++++++++++++++++++++++++++++ lib/routes/guozaoke/namespace.ts | 6 ++ 3 files changed, 159 insertions(+) create mode 100644 lib/routes/guozaoke/index.ts create mode 100644 lib/routes/guozaoke/namespace.ts diff --git a/lib/config.ts b/lib/config.ts index e2e0e162f8794f..191690754e43fb 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -338,6 +338,9 @@ export type Config = { zsxq: { accessToken?: string; }; + guozaoke: { + cookies?: string; + }; }; const value: Config | Record = {}; @@ -731,6 +734,9 @@ const calculateValue = () => { zsxq: { accessToken: envs.ZSXQ_ACCESS_TOKEN, }, + guozaoke: { + cookies: envs.GUOZAOKE_COOKIES, + }, }; for (const name in _value) { diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts new file mode 100644 index 00000000000000..6e2fc9e587f610 --- /dev/null +++ b/lib/routes/guozaoke/index.ts @@ -0,0 +1,147 @@ +import { Route } from '@/types'; +import got from '@/utils/got'; +import * as cheerio from 'cheerio'; +import { parseDate } from '@/utils/parse-date'; +import { config } from '@/config'; +import logger from '@/utils/logger'; + +export const route: Route = { + path: '', + categories: ['bbs'], + example: '/guozaoke', + parameters: {}, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'guozaoke', + maintainers: ['xiaoshame'], + handler, + url: 'guozaoke.com/', +}; + +interface Item { + title: string; + link: string; + author: string; + time: Date; +} + +interface ProcessedItem { + title: string; + link: string; + pubDate: Date; + description: string; + author: string; +} + +function convertToDate(relativeTime: string) { + const minutesAgoMatch = relativeTime.match(/\d+/); + const minutesAgo = minutesAgoMatch ? Number.parseInt(minutesAgoMatch[0], 10) : 0; + const now: number = Date.now(); + const pastDate = new Date(now - minutesAgo * 60 * 1000); // Subtract minutes in milliseconds + return pastDate; +} + +async function getContent(link) { + const url = `https://www.guozaoke.com${link}`; + const cookie = config.guozaoke.cookies; + const res = await got({ + method: 'get', + url, + headers: { + Cookie: cookie, + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36', + }, + }); + + const $ = cheerio.load(res.data); + let content = $('div.ui-content').html(); + content = content ? content.trim() : ''; + const comments = $('.reply-item') + .map((i, el) => { + const $el = $(el); + return { + comment: $el.find('span.content').text().trim(), // 提取content内容并去除前后空格 + author: $el.find('span.username').text(), // 提取username + }; + }) + .get(); + if (comments && comments.length > 0) { + for (const item of comments) { + content += '
' + item.author + ': ' + item.comment; + } + } + + return content ? content : ''; +} + +async function fetchContent(item: Item): Promise { + try { + const content = await getContent(item.link); + if (content === '') { + return null; // 如果内容为空，则返回null + } + return { + title: item.title, + link: item.link, + pubDate: parseDate(item.time), + description: content, + author: item.author, + }; + } catch (error) { + logger.error(error); + return null; // 如果发生错误，则返回null + } +} +async function processItems(itemsToFetch: Item[]): Promise { + const out: ProcessedItem[] = []; + let currentIndex = 0; + + // 使用索引直接访问原始数组，避免不必要的 slice 操作（可选优化） + const batchPromises: Promise[] = []; + for (let i = 0; i < 10 && currentIndex < itemsToFetch.length; i++, currentIndex++) { + batchPromises.push(fetchContent(itemsToFetch[currentIndex])); + } + const results = await Promise.all(batchPromises); + out.push(...results.filter((result): result is ProcessedItem => result !== null)); + + return out; +} + +async function handler() { + const url = `https://www.guozaoke.com/`; + const res = await got.get(url); + const $ = cheerio.load(res.body); + + const list = $('div.topic-item'); + const itemsToFetch: Item[] = []; + const maxItems = 10; // 最多取10个数据 + + for (const item of list) { + if (itemsToFetch.length >= maxItems) { + break; + } + const $item = $(item); + const title = $item.find('h3.title a').text(); + const link = $item.find('h3.title a').attr('href'); + const author = $item.find('span.username a').text(); + const lastTouched = $item.find('span.last-touched').text(); + const time = convertToDate(lastTouched); + if (link) { + itemsToFetch.push({ title, link, author, time }); + } + } + + const out = await processItems(itemsToFetch); + + return { + title: `过早客`, + link: url, + item: out, + }; +} diff --git a/lib/routes/guozaoke/namespace.ts b/lib/routes/guozaoke/namespace.ts new file mode 100644 index 00000000000000..386ab893f88126 --- /dev/null +++ b/lib/routes/guozaoke/namespace.ts @@ -0,0 +1,6 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'guozaoke', + url: 'guozaoke.com', +}; From 6a8e383310d2f5344a715a7d3a85ce3797c45591 Mon Sep 17 00:00:00 2001 From: asong Date: Thu, 17 Oct 2024 13:46:39 +0800 Subject: [PATCH 02/18] add routes guozaoke --- lib/routes/guozaoke/index.ts | 37 ++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 6e2fc9e587f610..bf03812abfcbe7 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -98,19 +98,36 @@ async function fetchContent(item: Item): Promise { return null; // 如果发生错误，则返回null } } -async function processItems(itemsToFetch: Item[]): Promise { - const out: ProcessedItem[] = []; - let currentIndex = 0; - // 使用索引直接访问原始数组，避免不必要的 slice 操作（可选优化） - const batchPromises: Promise[] = []; - for (let i = 0; i < 10 && currentIndex < itemsToFetch.length; i++, currentIndex++) { - batchPromises.push(fetchContent(itemsToFetch[currentIndex])); +// 递归处理函数，每次处理一批项 +async function processBatch(items: Item[], batchSize: number, out: ProcessedItem[], startIndex: number): Promise { + if (startIndex >= items.length) { + // 所有项都已处理完毕，返回结果 + return out; } + + // 确定这一批要处理的项数 + const endIndex = Math.min(startIndex + batchSize, items.length); + const batchItems = items.slice(startIndex, endIndex); + + // 创建这一批的 Promise 数组 + const batchPromises = batchItems.map((item) => fetchContent(item)); + + // 等待这一批 Promise 完成，并过滤结果 const results = await Promise.all(batchPromises); - out.push(...results.filter((result): result is ProcessedItem => result !== null)); + const filteredResults = results.filter((result): result is ProcessedItem => result !== null); - return out; + // 将过滤后的结果添加到输出数组中 + out.push(...filteredResults); + + // 递归处理下一批项 + return processBatch(items, batchSize, out, endIndex); +} + +function processItems(itemsToFetch: Item[]): Promise { + const batchSize = 2; // 并发请求的数量 + const out: ProcessedItem[] = []; + return processBatch(itemsToFetch, batchSize, out, 0); } async function handler() { @@ -120,7 +137,7 @@ async function handler() { const list = $('div.topic-item'); const itemsToFetch: Item[] = []; - const maxItems = 10; // 最多取10个数据 + const maxItems = 20; // 最多取20个数据 for (const item of list) { if (itemsToFetch.length >= maxItems) { From 3c57eec5206249ae0ffc42fc844e9bb83a626738 Mon Sep 17 00:00:00 2001 From: asong Date: Thu, 17 Oct 2024 13:55:35 +0800 Subject: [PATCH 03/18] fix ESLint Check warning --- lib/routes/guozaoke/index.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index bf03812abfcbe7..bfeb945abdd338 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -65,9 +65,11 @@ async function getContent(link) { const comments = $('.reply-item') .map((i, el) => { const $el = $(el); + const comment = $el.find('span.content').text().trim(); + const author = $el.find('span.username').text(); return { - comment: $el.find('span.content').text().trim(), // 提取content内容并去除前后空格 - author: $el.find('span.username').text(), // 提取username + comment, + author, }; }) .get(); @@ -77,13 +79,13 @@ async function getContent(link) { } } - return content ? content : ''; + return content; } async function fetchContent(item: Item): Promise { try { const content = await getContent(item.link); - if (content === '') { + if (content === undefined || content === '') { return null; // 如果内容为空，则返回null } return { From 8a1eb1de6423094054a7d18a489b981371689699 Mon Sep 17 00:00:00 2001 From: asong Date: Thu, 17 Oct 2024 13:59:56 +0800 Subject: [PATCH 04/18] fix ESLint Check warning --- lib/routes/guozaoke/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index bfeb945abdd338..5f6531193a2380 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -72,7 +72,7 @@ async function getContent(link) { author, }; }) - .get(); + .toArray(); if (comments && comments.length > 0) { for (const item of comments) { content += '
' + item.author + ': ' + item.comment; From 2583504daf9c91c916d47cdee43bb7b0a860e6f8 Mon Sep 17 00:00:00 2001 From: asong Date: Thu, 17 Oct 2024 14:06:13 +0800 Subject: [PATCH 05/18] fix ESLint Check warning --- lib/routes/guozaoke/index.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 5f6531193a2380..ac42bd682264db 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -62,17 +62,15 @@ async function getContent(link) { const $ = cheerio.load(res.data); let content = $('div.ui-content').html(); content = content ? content.trim() : ''; - const comments = $('.reply-item') - .map((i, el) => { - const $el = $(el); - const comment = $el.find('span.content').text().trim(); - const author = $el.find('span.username').text(); - return { - comment, - author, - }; - }) - .toArray(); + const comments = $('.reply-item').map((i, el) => { + const $el = $(el); + const comment = $el.find('span.content').text().trim(); + const author = $el.find('span.username').text(); + return { + comment, + author, + }; + }); if (comments && comments.length > 0) { for (const item of comments) { content += '
' + item.author + ': ' + item.comment; From 46eebb0717094310f9d7ce3792f590a4da4bb971 Mon Sep 17 00:00:00 2001 From: asong Date: Thu, 17 Oct 2024 14:09:51 +0800 Subject: [PATCH 06/18] fix ESLint Check warning --- lib/routes/guozaoke/index.ts | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index ac42bd682264db..5f6531193a2380 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -62,15 +62,17 @@ async function getContent(link) { const $ = cheerio.load(res.data); let content = $('div.ui-content').html(); content = content ? content.trim() : ''; - const comments = $('.reply-item').map((i, el) => { - const $el = $(el); - const comment = $el.find('span.content').text().trim(); - const author = $el.find('span.username').text(); - return { - comment, - author, - }; - }); + const comments = $('.reply-item') + .map((i, el) => { + const $el = $(el); + const comment = $el.find('span.content').text().trim(); + const author = $el.find('span.username').text(); + return { + comment, + author, + }; + }) + .toArray(); if (comments && comments.length > 0) { for (const item of comments) { content += '
' + item.author + ': ' + item.comment; From 31d78af6819d3e4b723cd9b50ac723f2520152ea Mon Sep 17 00:00:00 2001 From: asong Date: Thu, 17 Oct 2024 14:13:08 +0800 Subject: [PATCH 07/18] fix ESLint Check warning --- lib/routes/guozaoke/index.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 5f6531193a2380..ac42bd682264db 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -62,17 +62,15 @@ async function getContent(link) { const $ = cheerio.load(res.data); let content = $('div.ui-content').html(); content = content ? content.trim() : ''; - const comments = $('.reply-item') - .map((i, el) => { - const $el = $(el); - const comment = $el.find('span.content').text().trim(); - const author = $el.find('span.username').text(); - return { - comment, - author, - }; - }) - .toArray(); + const comments = $('.reply-item').map((i, el) => { + const $el = $(el); + const comment = $el.find('span.content').text().trim(); + const author = $el.find('span.username').text(); + return { + comment, + author, + }; + }); if (comments && comments.length > 0) { for (const item of comments) { content += '
' + item.author + ': ' + item.comment; From 5c0985a1c031815b67cbead0e75b5e8394ebd4bd Mon Sep 17 00:00:00 2001 From: asong Date: Fri, 25 Oct 2024 14:37:23 +0800 Subject: [PATCH 08/18] fix routes guozaoke code review --- lib/config.ts | 12 ++--- lib/routes/guozaoke/index.ts | 98 ++++++++++++++++++------------------ 2 files changed, 54 insertions(+), 56 deletions(-) diff --git a/lib/config.ts b/lib/config.ts index 9cb24beea5ea32..5daedff66279ae 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -151,6 +151,9 @@ export type Config = { google: { fontsApiKey?: string; }; + guozaoke: { + cookies?: string; + }; hefeng: { key?: string; }; @@ -348,9 +351,6 @@ export type Config = { zsxq: { accessToken?: string; }; - guozaoke: { - cookies?: string; - }; }; const value: Config | Record = {}; @@ -552,6 +552,9 @@ const calculateValue = () => { google: { fontsApiKey: envs.GOOGLE_FONTS_API_KEY, }, + guozaoke: { + cookies: envs.GUOZAOKE_COOKIES, + }, hefeng: { // weather key: envs.HEFENG_KEY, @@ -754,9 +757,6 @@ const calculateValue = () => { zsxq: { accessToken: envs.ZSXQ_ACCESS_TOKEN, }, - guozaoke: { - cookies: envs.GUOZAOKE_COOKIES, - }, }; for (const name in _value) { diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index ac42bd682264db..b1bccf2292dcff 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -1,9 +1,10 @@ import { Route } from '@/types'; import got from '@/utils/got'; -import * as cheerio from 'cheerio'; -import { parseDate } from '@/utils/parse-date'; +import { load } from 'cheerio'; +import { parseDate, parseRelativeDate } from '@/utils/parse-date'; import { config } from '@/config'; import logger from '@/utils/logger'; +import cache from '@/utils/cache'; export const route: Route = { path: '', @@ -28,38 +29,29 @@ interface Item { title: string; link: string; author: string; - time: Date; + time: string; } interface ProcessedItem { title: string; link: string; - pubDate: Date; + pubDate: string; description: string; author: string; } -function convertToDate(relativeTime: string) { - const minutesAgoMatch = relativeTime.match(/\d+/); - const minutesAgo = minutesAgoMatch ? Number.parseInt(minutesAgoMatch[0], 10) : 0; - const now: number = Date.now(); - const pastDate = new Date(now - minutesAgo * 60 * 1000); // Subtract minutes in milliseconds - return pastDate; -} - async function getContent(link) { const url = `https://www.guozaoke.com${link}`; - const cookie = config.guozaoke.cookies; const res = await got({ method: 'get', url, headers: { - Cookie: cookie, - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36', + Cookie: config.guozaoke.cookies, + 'User-Agent': config.ua, }, }); - const $ = cheerio.load(res.data); + const $ = load(res.data); let content = $('div.ui-content').html(); content = content ? content.trim() : ''; const comments = $('.reply-item').map((i, el) => { @@ -81,22 +73,24 @@ async function getContent(link) { } async function fetchContent(item: Item): Promise { - try { - const content = await getContent(item.link); - if (content === undefined || content === '') { - return null; // 如果内容为空，则返回null + return await cache.tryGet(item.link, async () => { + try { + const content = await getContent(item.link); + if (content === undefined || content === '') { + return null; // 如果内容为空，则返回null + } + return { + title: item.title, + link: item.link, + pubDate: parseDate(item.time), + description: content, + author: item.author, + }; + } catch (error) { + logger.error(error); + return null; // 如果发生错误，则返回null } - return { - title: item.title, - link: item.link, - pubDate: parseDate(item.time), - description: content, - author: item.author, - }; - } catch (error) { - logger.error(error); - return null; // 如果发生错误，则返回null - } + }); } // 递归处理函数，每次处理一批项 @@ -132,32 +126,36 @@ function processItems(itemsToFetch: Item[]): Promise { async function handler() { const url = `https://www.guozaoke.com/`; - const res = await got.get(url); - const $ = cheerio.load(res.body); + const res = await got({ + method: 'get', + url, + headers: { + Cookie: config.guozaoke.cookies, + 'User-Agent': config.ua, + }, + }); + const $ = load(res.data); - const list = $('div.topic-item'); - const itemsToFetch: Item[] = []; + const list = $('div.topic-item').toArray(); const maxItems = 20; // 最多取20个数据 - for (const item of list) { - if (itemsToFetch.length >= maxItems) { - break; - } - const $item = $(item); - const title = $item.find('h3.title a').text(); - const link = $item.find('h3.title a').attr('href'); - const author = $item.find('span.username a').text(); - const lastTouched = $item.find('span.last-touched').text(); - const time = convertToDate(lastTouched); - if (link) { - itemsToFetch.push({ title, link, author, time }); - } - } + const itemsToFetch: Item[] = list + .slice(0, maxItems) + .map((item) => { + const $item = $(item); + const title = $item.find('h3.title a').text(); + const link = $item.find('h3.title a').attr('href'); + const author = $item.find('span.username a').text(); + const lastTouched = $item.find('span.last-touched').text(); + const time = parseRelativeDate(lastTouched); + return link ? { title, link, author, time } : undefined; + }) + .filter((item) => item !== undefined); const out = await processItems(itemsToFetch); return { - title: `过早客`, + title: '过早客', link: url, item: out, }; From ab573d9741197c6da13df00ca6a53790bbfd0deb Mon Sep 17 00:00:00 2001 From: asong Date: Fri, 25 Oct 2024 15:27:34 +0800 Subject: [PATCH 09/18] fix routes guozaoke code review --- lib/routes/guozaoke/index.ts | 159 +++++++++++------------------------ 1 file changed, 49 insertions(+), 110 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index b1bccf2292dcff..6a1745cceb4cb9 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -1,10 +1,10 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { parseDate, parseRelativeDate } from '@/utils/parse-date'; +import { parseRelativeDate } from '@/utils/parse-date'; import { config } from '@/config'; -import logger from '@/utils/logger'; import cache from '@/utils/cache'; +import asyncPool from 'tiny-async-pool'; export const route: Route = { path: '', @@ -25,103 +25,38 @@ export const route: Route = { url: 'guozaoke.com/', }; -interface Item { - title: string; - link: string; - author: string; - time: string; -} - -interface ProcessedItem { - title: string; - link: string; - pubDate: string; - description: string; - author: string; -} - async function getContent(link) { - const url = `https://www.guozaoke.com${link}`; - const res = await got({ - method: 'get', - url, - headers: { - Cookie: config.guozaoke.cookies, - 'User-Agent': config.ua, - }, - }); - - const $ = load(res.data); - let content = $('div.ui-content').html(); - content = content ? content.trim() : ''; - const comments = $('.reply-item').map((i, el) => { - const $el = $(el); - const comment = $el.find('span.content').text().trim(); - const author = $el.find('span.username').text(); - return { - comment, - author, - }; - }); - if (comments && comments.length > 0) { - for (const item of comments) { - content += '
' + item.author + ': ' + item.comment; - } - } - - return content; -} - -async function fetchContent(item: Item): Promise { - return await cache.tryGet(item.link, async () => { - try { - const content = await getContent(item.link); - if (content === undefined || content === '') { - return null; // 如果内容为空，则返回null - } + return await cache.tryGet(link, async () => { + const url = `https://www.guozaoke.com${link}`; + const res = await got({ + method: 'get', + url, + headers: { + Cookie: config.guozaoke.cookies, + 'User-Agent': config.ua, + }, + }); + + const $ = load(res.data); + let content = $('div.ui-content').html(); + content = content ? content.trim() : ''; + const comments = $('.reply-item').map((i, el) => { + const $el = $(el); + const comment = $el.find('span.content').text().trim(); + const author = $el.find('span.username').text(); return { - title: item.title, - link: item.link, - pubDate: parseDate(item.time), - description: content, - author: item.author, + comment, + author, }; - } catch (error) { - logger.error(error); - return null; // 如果发生错误，则返回null + }); + if (comments && comments.length > 0) { + for (const item of comments) { + content += '
' + item.author + ': ' + item.comment; + } } - }); -} - -// 递归处理函数，每次处理一批项 -async function processBatch(items: Item[], batchSize: number, out: ProcessedItem[], startIndex: number): Promise { - if (startIndex >= items.length) { - // 所有项都已处理完毕，返回结果 - return out; - } - - // 确定这一批要处理的项数 - const endIndex = Math.min(startIndex + batchSize, items.length); - const batchItems = items.slice(startIndex, endIndex); - - // 创建这一批的 Promise 数组 - const batchPromises = batchItems.map((item) => fetchContent(item)); - - // 等待这一批 Promise 完成，并过滤结果 - const results = await Promise.all(batchPromises); - const filteredResults = results.filter((result): result is ProcessedItem => result !== null); - // 将过滤后的结果添加到输出数组中 - out.push(...filteredResults); - - // 递归处理下一批项 - return processBatch(items, batchSize, out, endIndex); -} - -function processItems(itemsToFetch: Item[]): Promise { - const batchSize = 2; // 并发请求的数量 - const out: ProcessedItem[] = []; - return processBatch(itemsToFetch, batchSize, out, 0); + return content; + }); } async function handler() { @@ -138,25 +73,29 @@ async function handler() { const list = $('div.topic-item').toArray(); const maxItems = 20; // 最多取20个数据 - - const itemsToFetch: Item[] = list - .slice(0, maxItems) - .map((item) => { - const $item = $(item); - const title = $item.find('h3.title a').text(); - const link = $item.find('h3.title a').attr('href'); - const author = $item.find('span.username a').text(); - const lastTouched = $item.find('span.last-touched').text(); - const time = parseRelativeDate(lastTouched); - return link ? { title, link, author, time } : undefined; - }) - .filter((item) => item !== undefined); - - const out = await processItems(itemsToFetch); + const items = []; + for await (const data of asyncPool(1, list.slice(0, maxItems), async (i) => { + const $item = $(i); + const title = $item.find('h3.title a').text(); + const link = $item.find('h3.title a').attr('href'); + const author = $item.find('span.username a').text(); + const lastTouched = $item.find('span.last-touched').text(); + const time = parseRelativeDate(lastTouched); + const description = await getContent(link); + return { + title, + link, + description, + author, + pubDate: time, + }; + })) { + items.push(data); + } return { title: '过早客', link: url, - item: out, + item: items, }; } From de3f6fc5a279907f7330c7971033b2e41de0654b Mon Sep 17 00:00:00 2001 From: asong Date: Fri, 25 Oct 2024 16:49:49 +0800 Subject: [PATCH 10/18] fix route code review --- lib/routes/guozaoke/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 6a1745cceb4cb9..0c990aed503557 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -84,7 +84,6 @@ async function handler() { const description = await getContent(link); return { title, - link, description, author, pubDate: time, From e18d6f71a9cdf2401010c5f9477f0759e9273cc1 Mon Sep 17 00:00:00 2001 From: asong Date: Fri, 25 Oct 2024 19:06:05 +0800 Subject: [PATCH 11/18] fix code revice issue --- lib/routes/guozaoke/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 0c990aed503557..b68a040caf702c 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -86,6 +86,7 @@ async function handler() { title, description, author, + link: link.split('#')[0], pubDate: time, }; })) { From a13fad6835b1891eb26acd260fcbbc3c67389f11 Mon Sep 17 00:00:00 2001 From: asong Date: Mon, 28 Oct 2024 17:14:10 +0800 Subject: [PATCH 12/18] fix code revice issue --- lib/routes/guozaoke/index.ts | 94 ++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index b68a040caf702c..7764b865d73155 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -7,7 +7,7 @@ import cache from '@/utils/cache'; import asyncPool from 'tiny-async-pool'; export const route: Route = { - path: '', + path: '/guozaoke', categories: ['bbs'], example: '/guozaoke', parameters: {}, @@ -25,40 +25,6 @@ export const route: Route = { url: 'guozaoke.com/', }; -async function getContent(link) { - return await cache.tryGet(link, async () => { - const url = `https://www.guozaoke.com${link}`; - const res = await got({ - method: 'get', - url, - headers: { - Cookie: config.guozaoke.cookies, - 'User-Agent': config.ua, - }, - }); - - const $ = load(res.data); - let content = $('div.ui-content').html(); - content = content ? content.trim() : ''; - const comments = $('.reply-item').map((i, el) => { - const $el = $(el); - const comment = $el.find('span.content').text().trim(); - const author = $el.find('span.username').text(); - return { - comment, - author, - }; - }); - if (comments && comments.length > 0) { - for (const item of comments) { - content += '
' + item.author + ': ' + item.comment; - } - } - - return content; - }); -} - async function handler() { const url = `https://www.guozaoke.com/`; const res = await got({ @@ -74,23 +40,57 @@ async function handler() { const list = $('div.topic-item').toArray(); const maxItems = 20; // 最多取20个数据 const items = []; - for await (const data of asyncPool(1, list.slice(0, maxItems), async (i) => { - const $item = $(i); + for (const item of list) { + if (items.length >= maxItems) { + break; + } + const $item = $(item); const title = $item.find('h3.title a').text(); const link = $item.find('h3.title a').attr('href'); const author = $item.find('span.username a').text(); const lastTouched = $item.find('span.last-touched').text(); const time = parseRelativeDate(lastTouched); - const description = await getContent(link); - return { - title, - description, - author, - link: link.split('#')[0], - pubDate: time, - }; - })) { - items.push(data); + if (link) { + items.push({ title, link, author, time }); + } + } + + const out = []; + for await (const result of asyncPool(2, items, (item) => + cache.tryGet(item.link, async () => { + const url = `https://www.guozaoke.com${item.link}`; + const res = await got({ + method: 'get', + url, + headers: { + Cookie: config.guozaoke.cookies, + 'User-Agent': config.ua, + }, + }); + + const $ = load(res.data); + let content = $('div.ui-content').html(); + content = content ? content.trim() : ''; + const comments = $('.reply-item').map((i, el) => { + const $el = $(el); + const comment = $el.find('span.content').text().trim(); + const author = $el.find('span.username').text(); + return { + comment, + author, + }; + }); + if (comments && comments.length > 0) { + for (const item of comments) { + content += '
' + item.author + ': ' + item.comment; + } + } + + item.link = item.link.split('#')[0]; + item.description = content; + }) + )) { + out.push(result); } return { From 6c1909fde5e09e2bb0a57423558c7a25c9522e21 Mon Sep 17 00:00:00 2001 From: asong Date: Mon, 28 Oct 2024 17:19:56 +0800 Subject: [PATCH 13/18] fix code revice issue --- lib/routes/guozaoke/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 7764b865d73155..ed51338b679af3 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -7,9 +7,9 @@ import cache from '@/utils/cache'; import asyncPool from 'tiny-async-pool'; export const route: Route = { - path: '/guozaoke', + path: '/default', categories: ['bbs'], - example: '/guozaoke', + example: '/guozaoke/default', parameters: {}, features: { requireConfig: false, From 7f8922043d798cf953e04d424bee84a84f5d2753 Mon Sep 17 00:00:00 2001 From: asong Date: Mon, 28 Oct 2024 17:23:47 +0800 Subject: [PATCH 14/18] fix code revice issue --- lib/routes/guozaoke/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index ed51338b679af3..f1cc22d78a67a3 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -7,9 +7,9 @@ import cache from '@/utils/cache'; import asyncPool from 'tiny-async-pool'; export const route: Route = { - path: '/default', + path: '/guozaoke', categories: ['bbs'], - example: '/guozaoke/default', + example: '/guozaoke', parameters: {}, features: { requireConfig: false, @@ -19,7 +19,7 @@ export const route: Route = { supportPodcast: false, supportScihub: false, }, - name: 'guozaoke', + name: '过早客', maintainers: ['xiaoshame'], handler, url: 'guozaoke.com/', From 556704cd27087ffa0b3133f29ee0a55a410e101f Mon Sep 17 00:00:00 2001 From: asong Date: Mon, 28 Oct 2024 17:30:43 +0800 Subject: [PATCH 15/18] fix code revice issue --- lib/routes/guozaoke/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index f1cc22d78a67a3..e63a1d10a3b86a 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -7,9 +7,9 @@ import cache from '@/utils/cache'; import asyncPool from 'tiny-async-pool'; export const route: Route = { - path: '/guozaoke', + path: '/default', categories: ['bbs'], - example: '/guozaoke', + example: '/guozaoke/default', parameters: {}, features: { requireConfig: false, From 90a6ffe1054f247f2e3b94d0013c8b4bd327416e Mon Sep 17 00:00:00 2001 From: asong Date: Mon, 28 Oct 2024 17:41:19 +0800 Subject: [PATCH 16/18] fix code revice issue --- lib/routes/guozaoke/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index e63a1d10a3b86a..7fb61215685946 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -88,6 +88,7 @@ async function handler() { item.link = item.link.split('#')[0]; item.description = content; + return item; }) )) { out.push(result); @@ -96,6 +97,6 @@ async function handler() { return { title: '过早客', link: url, - item: items, + item: out, }; } From ef5050f5e140684bfaa58259784bb7f7e42b9730 Mon Sep 17 00:00:00 2001 From: asong Date: Tue, 29 Oct 2024 08:36:19 +0800 Subject: [PATCH 17/18] fix code review issues --- lib/routes/guozaoke/index.ts | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 7fb61215685946..9536f58c730f55 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -26,7 +26,7 @@ export const route: Route = { }; async function handler() { - const url = `https://www.guozaoke.com/`; + const url = 'https://www.guozaoke.com/'; const res = await got({ method: 'get', url, @@ -39,21 +39,20 @@ async function handler() { const list = $('div.topic-item').toArray(); const maxItems = 20; // 最多取20个数据 - const items = []; - for (const item of list) { - if (items.length >= maxItems) { - break; - } - const $item = $(item); - const title = $item.find('h3.title a').text(); - const link = $item.find('h3.title a').attr('href'); - const author = $item.find('span.username a').text(); - const lastTouched = $item.find('span.last-touched').text(); - const time = parseRelativeDate(lastTouched); - if (link) { - items.push({ title, link, author, time }); - } - } + + const items = list + .slice(0, maxItems) + .map((item) => { + const $item = $(item); + const title = $item.find('h3.title a').text(); + const url = $item.find('h3.title a').attr('href'); + const author = $item.find('span.username a').text(); + const lastTouched = $item.find('span.last-touched').text(); + const time = parseRelativeDate(lastTouched); + const link = url ? url.split('#')[0] : undefined; + return link ? { title, link, author, time } : undefined; + }) + .filter((item) => item !== undefined); const out = []; for await (const result of asyncPool(2, items, (item) => @@ -85,8 +84,6 @@ async function handler() { content += '
' + item.author + ': ' + item.comment; } } - - item.link = item.link.split('#')[0]; item.description = content; return item; }) From 135b9d3432777d816eb92943c494d1aebbfae1b5 Mon Sep 17 00:00:00 2001 From: asong Date: Wed, 30 Oct 2024 09:45:49 +0800 Subject: [PATCH 18/18] fix code review issues --- lib/routes/guozaoke/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/routes/guozaoke/index.ts b/lib/routes/guozaoke/index.ts index 9536f58c730f55..d8fe2c35e0a848 100644 --- a/lib/routes/guozaoke/index.ts +++ b/lib/routes/guozaoke/index.ts @@ -48,9 +48,9 @@ async function handler() { const url = $item.find('h3.title a').attr('href'); const author = $item.find('span.username a').text(); const lastTouched = $item.find('span.last-touched').text(); - const time = parseRelativeDate(lastTouched); + const pubDate = parseRelativeDate(lastTouched); const link = url ? url.split('#')[0] : undefined; - return link ? { title, link, author, time } : undefined; + return link ? { title, link, author, pubDate } : undefined; }) .filter((item) => item !== undefined);