From 3a64e9eb91a7db183eaf72dafc45732f8ea6302d Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Wed, 9 Oct 2024 00:21:50 +0800 Subject: [PATCH] =?UTF-8?q?feat(route):=20add=20=E4=B8=AD=E5=8D=8E?= =?UTF-8?q?=E4=BA=BA=E6=B0=91=E5=85=B1=E5=92=8C=E5=9B=BD=E5=9B=BD=E5=AE=B6?= =?UTF-8?q?=E5=8F=91=E5=B1=95=E5=92=8C=E6=94=B9=E9=9D=A9=E5=A7=94=E5=91=98?= =?UTF-8?q?=E4=BC=9A=E6=94=BF=E5=BA=9C=E4=BF=A1=E6=81=AF=E5=85=AC=E5=BC=80?= =?UTF-8?q?=20(#16850)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(route): add 中华人民共和国国家发展和改革委员会政府信息公开 * update to existing route * feat(route): add 中华人民共和国国家发展和改革委员会政府信息公开 --- lib/routes/gov/ndrc/zfxxgk.ts | 106 +++++++++++++++++++++++++ lib/routes/gov/ndrc/zfxxgk/articles.ts | 73 ----------------- 2 files changed, 106 insertions(+), 73 deletions(-) create mode 100644 lib/routes/gov/ndrc/zfxxgk.ts delete mode 100644 lib/routes/gov/ndrc/zfxxgk/articles.ts diff --git a/lib/routes/gov/ndrc/zfxxgk.ts b/lib/routes/gov/ndrc/zfxxgk.ts new file mode 100644 index 00000000000000..72e274ba356b09 --- /dev/null +++ b/lib/routes/gov/ndrc/zfxxgk.ts @@ -0,0 +1,106 @@ +import { Route } from '@/types'; + +import cache from '@/utils/cache'; +import got from '@/utils/got'; +import { load } from 'cheerio'; +import { parseDate } from '@/utils/parse-date'; + +export const handler = async (ctx) => { + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 15; + + const rootUrl = 'https://zfxxgk.ndrc.gov.cn'; + const currentUrl = new URL('web/dirlist.jsp', rootUrl).href; + + const { data: response } = await got(currentUrl); + + const $ = load(response); + + const language = $('html').prop('lang'); + + $('th').parent().remove(); + + let items = $('div.zwxxkg-result tr') + .slice(0, limit) + .toArray() + .map((item) => { + item = $(item); + + const a = item.find('a.xxgk_list1'); + + return { + title: a.text(), + pubDate: parseDate(item.find('td').last().text()), + link: new URL(a.prop('href'), currentUrl).href, + language, + }; + }); + + items = await Promise.all( + items.map((item) => + cache.tryGet(item.link, async () => { + const { data: detailResponse } = await got(item.link); + + const $$ = load(detailResponse); + + const title = $$('meta[name="ArticleTitle"]').prop('content'); + const description = $$('div.zwgkdetail').html(); + + item.title = title; + item.description = description; + item.category = [...new Set([$$('meta[name="ContentSource"]').prop('content'), $$('meta[name="ColumnName"]').prop('content'), $$('meta[name="ColumnType"]').prop('content')])].filter(Boolean); + item.author = $$('meta[name="ContentSource"]').prop('content'); + item.content = { + html: description, + text: $$('div.article').text(), + }; + item.language = language; + item.enclosure_url = $$('table.enclosure a.xxgk_list1').length === 0 ? undefined : new URL($$('table.enclosure a.xxgk_list1').first().prop('href'), currentUrl).href; + item.enclosure_title = item.enclosure_url ? $$('table.enclosure a.xxgk_list1').first().text() : undefined; + + return item; + }) + ) + ); + + const image = new URL($('div.zwgklogo img').prop('src'), currentUrl).href; + + return { + title: `${$('meta[name="SiteName"]').prop('content')} - ${$('div.zwgktoptitle').text()}`, + description: $('meta[name="ColumnDescription"]').prop('content'), + link: currentUrl, + item: items, + allowEmpty: true, + image, + author: $('meta[name="SiteName"]').prop('content'), + language, + }; +}; + +export const route: Route = { + path: ['/ndrc/zfxxgk'], + // path: ['/ndrc/zfxxgk', '/ndrc/zfxxgk/iteminfo'], + name: '中华人民共和国国家发展和改革委员会政府信息公开', + url: 'zfxxgk.ndrc.gov.cn', + maintainers: ['howfool', 'nczitzk'], + handler, + example: '/gov/ndrc/zfxxgk', + parameters: undefined, + description: undefined, + categories: ['government'], + + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportRadar: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['zfxxgk.ndrc.gov.cn/web/dirlist.jsp'], + target: '/ndrc/zfxxgk', + }, + ], +}; diff --git a/lib/routes/gov/ndrc/zfxxgk/articles.ts b/lib/routes/gov/ndrc/zfxxgk/articles.ts deleted file mode 100644 index 12304cc6842fc0..00000000000000 --- a/lib/routes/gov/ndrc/zfxxgk/articles.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import { load } from 'cheerio'; -import { parseDate } from '@/utils/parse-date'; - -export const route: Route = { - path: '/ndrc/zfxxgk/iteminfo', - categories: ['government'], - example: '/gov/ndrc/zfxxgk/iteminfo', - parameters: {}, - features: { - requireConfig: false, - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, - name: '国家发展改革委 - 政府信息公开', - maintainers: ['howfool'], - handler, - url: 'zfxxgk.ndrc.gov.cn/web/dirlist.jsp', -}; - -async function handler() { - const homeUrl = 'https://zfxxgk.ndrc.gov.cn/web/dirlist.jsp'; - const rootUrl = 'https://zfxxgk.ndrc.gov.cn/web/'; - - const response = await got({ - method: 'get', - url: homeUrl, - }); - - const $ = load(response.data); - - let items = $('div.zwgk-right .zwxxkg-result tr') - .toArray() - .slice(1) - .map((item) => { - item = $(item); - - return { - title: item.find('a').text().replace(/^\s*/, ''), - link: rootUrl + item.find('a').attr('href'), - data: item.find('td:last').text(), - }; - }); - - items = await Promise.all( - items.map((item) => - cache.tryGet(item.link, async () => { - const detailResponse = await got({ - method: 'get', - url: item.link, - }); - - const content = load(detailResponse.data); - content('[class$="top"]').remove(); - item.description = content('.zwgkbg').html(); - item.pubDate = parseDate(item.data); - - return item; - }) - ) - ); - - return { - title: '国家发展改革委 - 政府信息公开', - link: homeUrl, - item: items, - }; -}