From ae280d1ccf0460aa162d97cfd2150b8f51e03894 Mon Sep 17 00:00:00 2001 From: nczitzk <42264778+nczitzk@users.noreply.github.com> Date: Tue, 2 Jan 2024 11:43:56 +0800 Subject: [PATCH 1/2] fix(route): Tradingview Blog --- lib/v2/tradingview/blog.js | 103 +++++++++++++------ lib/v2/tradingview/maintainer.js | 2 +- lib/v2/tradingview/radar.js | 88 +++++++++++++++- lib/v2/tradingview/router.js | 2 +- lib/v2/tradingview/templates/description.art | 15 ++- website/docs/routes/program-update.mdx | 23 ++++- 6 files changed, 193 insertions(+), 40 deletions(-) diff --git a/lib/v2/tradingview/blog.js b/lib/v2/tradingview/blog.js index bbac1845fb3e2e..689effa31a19e5 100644 --- a/lib/v2/tradingview/blog.js +++ b/lib/v2/tradingview/blog.js @@ -1,60 +1,101 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); -const asyncPool = require('tiny-async-pool'); const { art } = require('@/utils/render'); const path = require('path'); module.exports = async (ctx) => { - const language = ctx.params.language ?? 'en'; + const { category = 'en' } = ctx.params; + const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 22; const rootUrl = 'https://www.tradingview.com'; - const currentUrl = `${rootUrl}/blog/${language}`; + const currentUrl = new URL(`blog/${category.endsWith('/') ? category : `${category}/`}`, rootUrl).href; - const response = await got({ - method: 'get', - url: currentUrl, - }); + const { data: response } = await got(currentUrl); - const $ = cheerio.load(response.data); + const $ = cheerio.load(response); - const list = $('.articles-grid-item a[rel="bookmark"]') - .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20) + let items = $('article[id]') + .slice(0, limit) .toArray() .map((item) => { item = $(item); + const title = item.find('div.title').text(); + return { - title: item.find('.title').text(), - link: item.attr('href'), - pubDate: parseDate(item.find('.date').text(), 'MMMM D, YYYY'), + title, + link: item.find('a.articles-grid-link').prop('href'), + description: art(path.join(__dirname, 'templates/description.art'), { + image: { + src: item + .find('div.articles-grid-img img') + .prop('src') + .replace(/-\d+x\d+\./, '.'), + alt: title, + }, + }), + category: item + .find('a.section') + .toArray() + .map((c) => $(c).text()), + guid: `tradingview-blog-${item.prop('id')}`, + pubDate: parseDate(item.find('div.date').text(), 'MMM D, YYYY'), }; }); - const items = []; - for await (const item of asyncPool(3, list, (item) => - ctx.cache.tryGet(item.link, async () => { - const detailResponse = await got({ - method: 'get', - url: item.link, - }); + items = await Promise.all( + items.map((item) => + ctx.cache.tryGet(item.link, async () => { + const { data: detailResponse } = await got(item.link); + + const content = cheerio.load(detailResponse); + + content('div.entry-content') + .find('img') + .each((_, e) => { + content(e).replaceWith( + art(path.join(__dirname, 'templates/description.art'), { + image: { + src: content(e) + .prop('src') + .replace(/-\d+x\d+\./, '.'), + width: content(e).prop('width'), + height: content(e).prop('height'), + }, + }) + ); + }); - const content = cheerio.load(detailResponse.data); + item.title = content('meta[property="og:title"]').prop('content'); + item.description = art(path.join(__dirname, 'templates/description.art'), { + image: { + src: content('meta[property="og:image"]').prop('content'), + alt: item.title, + }, + description: content('div.entry-content').html(), + }); + item.author = content('meta[property="og:site_name"]').prop('content'); + item.category = content('div.sections a.section') + .toArray() + .map((c) => content(c).text()); + item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY'); - item.description = art(path.join(__dirname, 'templates/description.art'), { - image: content('.single-img img').attr('src'), - description: content('.entry-content').html(), - }); + return item; + }) + ) + ); - return item; - }) - )) { - items.push(item); - } + const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href; ctx.state.data = { + item: items, title: $('title').text(), link: currentUrl, - item: items, + description: $('div.site-subtitle').text(), + language: $('html').prop('lang'), + icon, + logo: icon, + subtitle: $('h1.site-title').text(), }; }; diff --git a/lib/v2/tradingview/maintainer.js b/lib/v2/tradingview/maintainer.js index 8135e010a57868..cb9d37b3a3a949 100644 --- a/lib/v2/tradingview/maintainer.js +++ b/lib/v2/tradingview/maintainer.js @@ -1,3 +1,3 @@ module.exports = { - '/blog/:language?': ['nczitzk'], + '/blog/:language?/category/:category?': ['nczitzk'], }; diff --git a/lib/v2/tradingview/radar.js b/lib/v2/tradingview/radar.js index 6dd935085aefd1..642caa3f60af72 100644 --- a/lib/v2/tradingview/radar.js +++ b/lib/v2/tradingview/radar.js @@ -5,8 +5,92 @@ module.exports = { { title: 'Blog', docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', - source: ['/blog/:language', '/'], - target: '/tradingview/blog', + source: ['/blog/:language/'], + target: '/tradingview/blog/:language/', + }, + { + title: 'Blog - Alerts', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/alerts/'], + target: '/tradingview/blog/:language/category/alerts', + }, + { + title: 'Blog - Bitcoin and Crypto', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/bitcoin-charts/'], + target: '/tradingview/blog/:language/category/bitcoin-charts', + }, + { + title: 'Blog - Business Updates', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/business-updates/'], + target: '/tradingview/blog/:language/category/business-updates', + }, + { + title: 'Blog - Charting', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/charts/'], + target: '/tradingview/blog/:language/category/charts', + }, + { + title: 'Blog - Charting Library', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/charting-library/'], + target: '/tradingview/blog/:language/category/charting-library', + }, + { + title: 'Blog - Data Feeds and Exchanges', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/data-feeds-exchanges/'], + target: '/tradingview/blog/:language/category/data-feeds-exchanges', + }, + { + title: 'Blog - Desktop', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/desktop/'], + target: '/tradingview/blog/:language/category/desktop', + }, + { + title: 'Blog - Market Analysis', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/market-analysis/'], + target: '/tradingview/blog/:language/category/market-analysis', + }, + { + title: 'Blog - Mobile', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/mobile/'], + target: '/tradingview/blog/:language/category/mobile', + }, + { + title: 'Blog - Pine Script®', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/pine/'], + target: '/tradingview/blog/:language/category/pine', + }, + { + title: 'Blog - Screener', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/stock-screener/'], + target: '/tradingview/blog/:language/category/stock-screener', + }, + { + title: 'Blog - Social', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/social/'], + target: '/tradingview/blog/:language/category/social', + }, + { + title: 'Blog - Trading and Brokerage', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/trading/'], + target: '/tradingview/blog/:language/category/trading', + }, + { + title: 'Blog - Widgets', + docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog', + source: ['/blog/:language/category/widgets/'], + target: '/tradingview/blog/:language/category/widgets', }, ], }, diff --git a/lib/v2/tradingview/router.js b/lib/v2/tradingview/router.js index fadb0f6b3767da..98bb22bf2c0d0a 100644 --- a/lib/v2/tradingview/router.js +++ b/lib/v2/tradingview/router.js @@ -1,3 +1,3 @@ module.exports = function (router) { - router.get('/blog/:language?', require('./blog')); + router.get('/blog/:category*', require('./blog')); }; diff --git a/lib/v2/tradingview/templates/description.art b/lib/v2/tradingview/templates/description.art index c823516c9a36ad..a89e118b25e33d 100644 --- a/lib/v2/tradingview/templates/description.art +++ b/lib/v2/tradingview/templates/description.art @@ -1,4 +1,13 @@ -{{ if image }} - +{{ if image?.src }} +
+ {{ image.alt }} +
{{ /if }} -{{@ description }} \ No newline at end of file + +{{ if description }} + {{@ description }} +{{ /if }} \ No newline at end of file diff --git a/website/docs/routes/program-update.mdx b/website/docs/routes/program-update.mdx index 03f96da3c8bd00..80e39d758b6d1e 100644 --- a/website/docs/routes/program-update.mdx +++ b/website/docs/routes/program-update.mdx @@ -564,8 +564,8 @@ Logseq 开发团队已经放弃了 [旧网站](https://logseq.com/blog)。 ### Blog {#tradingview-blog} - - Language + + #### Language | Id | Language | | -- | ------------------- | @@ -589,6 +589,25 @@ Logseq 开发团队已经放弃了 [旧网站](https://logseq.com/blog)。 | sv | Svenska | | ar | العربية | | il | Hebrew | + + #### Category + + | Category | ID | + | ---------------------------------------------------------------------------------------------- | ----------------------------- | + | [Alerts](https://www.tradingview.com/blog/en/category/alerts/) | category/alerts | + | [Bitcoin and Crypto](https://www.tradingview.com/blog/en/category/bitcoin-charts/) | category/bitcoin-charts | + | [Business Updates](https://www.tradingview.com/blog/en/category/business-updates/) | category/business-updates | + | [Charting](https://www.tradingview.com/blog/en/category/charts/) | category/charts | + | [Charting Library](https://www.tradingview.com/blog/en/category/charting-library/) | category/charting-library | + | [Data Feeds and Exchanges](https://www.tradingview.com/blog/en/category/data-feeds-exchanges/) | category/data-feeds-exchanges | + | [Desktop](https://www.tradingview.com/blog/en/category/desktop/) | category/desktop | + | [Market Analysis](https://www.tradingview.com/blog/en/category/market-analysis/) | category/market-analysis | + | [Mobile](https://www.tradingview.com/blog/en/category/mobile/) | category/mobile | + | [Pine Script®](https://www.tradingview.com/blog/en/category/pine/) | category/pine | + | [Screener](https://www.tradingview.com/blog/en/category/stock-screener/) | category/stock-screener | + | [Social](https://www.tradingview.com/blog/en/category/social/) | category/social | + | [Trading and Brokerage](https://www.tradingview.com/blog/en/category/trading/) | category/trading | + | [Widgets](https://www.tradingview.com/blog/en/category/widgets/) | category/widgets | ## Typora {#typora} From 5add23295d17e5614129f973c3b102b983acfa22 Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Sat, 6 Jan 2024 11:25:14 +0800 Subject: [PATCH 2/2] fix: use tiny-async-pool --- lib/v2/tradingview/blog.js | 80 +++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/lib/v2/tradingview/blog.js b/lib/v2/tradingview/blog.js index 689effa31a19e5..2efb7889b9ba66 100644 --- a/lib/v2/tradingview/blog.js +++ b/lib/v2/tradingview/blog.js @@ -1,6 +1,7 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); +const asyncPool = require('tiny-async-pool'); const { art } = require('@/utils/render'); const path = require('path'); @@ -15,7 +16,7 @@ module.exports = async (ctx) => { const $ = cheerio.load(response); - let items = $('article[id]') + const items = $('article[id]') .slice(0, limit) .toArray() .map((item) => { @@ -39,52 +40,53 @@ module.exports = async (ctx) => { .find('a.section') .toArray() .map((c) => $(c).text()), - guid: `tradingview-blog-${item.prop('id')}`, + guid: `tradingview-blog-${category}-${item.prop('id')}`, pubDate: parseDate(item.find('div.date').text(), 'MMM D, YYYY'), }; }); - items = await Promise.all( - items.map((item) => - ctx.cache.tryGet(item.link, async () => { - const { data: detailResponse } = await got(item.link); + for await (const item of asyncPool(3, items, (item) => + ctx.cache.tryGet(item.link, async () => { + const { data: detailResponse } = await got(item.link); - const content = cheerio.load(detailResponse); + const content = cheerio.load(detailResponse); - content('div.entry-content') - .find('img') - .each((_, e) => { - content(e).replaceWith( - art(path.join(__dirname, 'templates/description.art'), { - image: { - src: content(e) - .prop('src') - .replace(/-\d+x\d+\./, '.'), - width: content(e).prop('width'), - height: content(e).prop('height'), - }, - }) - ); - }); - - item.title = content('meta[property="og:title"]').prop('content'); - item.description = art(path.join(__dirname, 'templates/description.art'), { - image: { - src: content('meta[property="og:image"]').prop('content'), - alt: item.title, - }, - description: content('div.entry-content').html(), + content('div.entry-content') + .find('img') + .each((_, e) => { + content(e).replaceWith( + art(path.join(__dirname, 'templates/description.art'), { + image: { + src: content(e) + .prop('src') + .replace(/-\d+x\d+\./, '.'), + width: content(e).prop('width'), + height: content(e).prop('height'), + }, + }) + ); }); - item.author = content('meta[property="og:site_name"]').prop('content'); - item.category = content('div.sections a.section') - .toArray() - .map((c) => content(c).text()); - item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY'); - return item; - }) - ) - ); + item.title = content('meta[property="og:title"]').prop('content'); + item.description = art(path.join(__dirname, 'templates/description.art'), { + image: { + src: content('meta[property="og:image"]').prop('content'), + alt: item.title, + }, + description: content('div.entry-content').html(), + }); + item.author = content('meta[property="og:site_name"]').prop('content'); + item.category = content('div.sections a.section') + .toArray() + .map((c) => content(c).text()); + item.pubDate = parseDate(content('div.single-date').text(), 'MMM D, YYYY'); + + return item; + }) + )) { + items.shift(); + items.push(item); + } const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href;