Skip to content

Commit

Permalink
fix(route): 抓取北极星风电网文章全文 (#16273)
Browse files Browse the repository at this point in the history
* fix(route): 抓取北极星风电网文章全文

* Update lib/routes/bjx/fd.ts

---------
  • Loading branch information
hualiong authored Jul 27, 2024
1 parent 4dd9837 commit 1754ffa
Showing 1 changed file with 35 additions and 26 deletions.
61 changes: 35 additions & 26 deletions lib/routes/bjx/fd.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
import { DataItem, Route } from '@/types';
import got from '@/utils/got';
import { load } from 'cheerio';
import { parseDate } from '@/utils/parse-date';
import ofetch from '@/utils/ofetch';
import cache from '@/utils/cache';

export const route: Route = {
path: '/fd/:type',
categories: ['traditional-media'],
example: '/bjx/fd/yw',
parameters: { type: '文章分类' },
parameters: { type: '文章分类,详见下表' },
features: {
requireConfig: false,
requirePuppeteer: false,
Expand All @@ -18,37 +19,45 @@ export const route: Route = {
},
name: '风电',
maintainers: ['hualiong'],
description: `\`:type\` 类型可选如下
| 要闻 | 政策 | 数据 | 市场 | 企业 | 招标 | 技术 | 报道 |
| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| yw | zc | sj | sc | mq | zb | js | bd |`,
handler: async (ctx) => {
const type = ctx.req.param('type');
const response = await got({
method: 'get',
url: `https://fd.bjx.com.cn/${type}/`,
});
const data = response.data;
const $ = load(data);
const typeName = $('div.box2 em:last').text();
const list = $('div.cc-list-content ul li');
const response = await ofetch(`https://fd.bjx.com.cn/${type}/`);

const $ = load(response);
const typeName = $('div.box2 em:last-child').text();
const list = $('div.cc-list-content ul li:nth-child(-n+20)')
.toArray()
.map((item): DataItem => {
const each = $(item);
return {
title: each.find('a').attr('title')!,
link: each.find('a').attr('href'),
pubDate: parseDate(each.find('span').text()),
};
});

const items = await Promise.all(
list.map((item) =>
cache.tryGet(item.link!, async () => {
const response = await ofetch(item.link!);
const $ = load(response);

item.description = $('#article_cont').html()!;
return item;
})
)
);

return {
title: `北极星风力发电网${typeName}`,
description: $('meta[name="Description"]').attr('content'),
link: `https://fd.bjx.com.cn/${type}/`,
item: list
.map((index, item) => {
const each = $(item);
return {
title: each.find('a').attr('title'),
description: each.html(),
link: each.find('a').attr('href'),
pubDate: parseDate(each.find('span').text()),
};
})
.get() as DataItem[],
item: items as DataItem[],
};
},
description: `\`:type\` 类型可选如下
| 要闻 | 政策 | 数据 | 市场 | 企业 | 招标 | 技术 | 报道 |
| ---- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
| yw | zc | sj | sc | mq | zb | js | bd |`,
};

0 comments on commit 1754ffa

Please sign in to comment.