Skip to content

Commit

Permalink
fix: Fixed the issue of being blocked by the website due to excessive… (
Browse files Browse the repository at this point in the history
#18049)

* fix: Fixed the issue of being blocked by the website due to excessive concurrent requests, which resulted in failure to retrieve information.

* fix: Added the missing semicolons.

* fix: use tiny-async-pool instead of resolving one by one.
  • Loading branch information
luguoj authored Jan 6, 2025
1 parent 8aff3ec commit 6d9ab1f
Showing 1 changed file with 19 additions and 17 deletions.
36 changes: 19 additions & 17 deletions lib/routes/agefans/update.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import cache from '@/utils/cache';
import got from '@/utils/got';
import { load } from 'cheerio';
import { rootUrl } from './utils';
import asyncPool from 'tiny-async-pool';

export const route: Route = {
path: '/update',
Expand Down Expand Up @@ -46,26 +47,27 @@ async function handler() {
};
});

const items = await Promise.all(
list.map((item) =>
cache.tryGet(item.link, async () => {
const detailResponse = await got(item.link);
const content = load(detailResponse.data);
const items:any[] = [];
for await (const item of asyncPool(3, list, (item) =>
cache.tryGet(item.link, async () => {
const detailResponse = await got(item.link);
const content = load(detailResponse.data);

content('img').each((_, ele) => {
if (ele.attribs['data-original']) {
ele.attribs.src = ele.attribs['data-original'];
delete ele.attribs['data-original'];
}
});
content('.video_detail_collect').remove();
content('img').each((_, ele) => {
if (ele.attribs['data-original']) {
ele.attribs.src = ele.attribs['data-original'];
delete ele.attribs['data-original'];
}
});
content('.video_detail_collect').remove();

item.description = content('.video_detail_left').html();
item.description = content('.video_detail_left').html();

return item;
})
)
);
return item;
})
)) {
items.push(item);
}

return {
title: $('title').text(),
Expand Down

0 comments on commit 6d9ab1f

Please sign in to comment.