From 823ddd655b19c05c2c3173416b731405de6f27ae Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:43:41 +0800 Subject: [PATCH 01/10] feat: ssrf check --- .github/workflows/docker-test-cont.yml | 1 - .github/workflows/issue-command.yml | 1 - lib/config.ts | 2 -- lib/routes-deprecated/gitlab/common.js | 5 ----- lib/routes-deprecated/gitlab/explore.js | 5 ----- lib/routes-deprecated/gitlab/release.js | 5 ----- lib/routes-deprecated/gitlab/tag.js | 5 ----- lib/routes-deprecated/hexo/fluid.js | 3 --- lib/routes-deprecated/hexo/next.js | 3 --- lib/routes-deprecated/hexo/yilia.js | 3 --- lib/routes.test.ts | 7 +------ lib/routes/18comic/utils.ts | 12 +----------- lib/routes/91porn/author.ts | 2 -- lib/routes/91porn/index.ts | 2 -- lib/routes/91porn/utils.ts | 11 ----------- lib/routes/bdys/index.ts | 8 -------- lib/routes/biquge/index.ts | 22 ---------------------- lib/routes/btzj/index.ts | 6 ------ lib/routes/domp4/detail.ts | 4 ++-- lib/routes/domp4/latest-movie-bt.ts | 4 ++-- lib/routes/domp4/latest.ts | 4 ++-- lib/routes/domp4/utils.ts | 15 +-------------- lib/routes/fediverse/timeline.ts | 6 ------ lib/routes/javbus/index.ts | 8 -------- lib/routes/javdb/utils.ts | 6 ------ lib/routes/lemmy/index.ts | 11 ----------- lib/routes/mastodon/account-id.ts | 5 ----- lib/routes/mastodon/timeline-local.ts | 6 ------ lib/routes/mastodon/timeline-remote.ts | 6 ------ lib/routes/mastodon/utils.ts | 3 --- lib/routes/misskey/featured-notes.ts | 5 ----- lib/routes/misskey/user-timeline.ts | 5 ----- lib/routes/rsshub/transform/html.ts | 11 ----------- lib/routes/rsshub/transform/json.ts | 11 ----------- lib/routes/rsshub/transform/sitemap.ts | 5 ----- lib/routes/wordpress/index.ts | 14 +------------- lib/utils/ofetch.ts | 19 +++++++++++++++++++ package.json | 1 + pnpm-lock.yaml | 9 +++++++++ 39 files changed, 39 insertions(+), 222 deletions(-) delete mode 100644 lib/routes-deprecated/gitlab/common.js delete mode 100644 lib/routes/91porn/utils.ts diff --git a/.github/workflows/docker-test-cont.yml b/.github/workflows/docker-test-cont.yml index 761706042c0d47..47f83ac92f7064 100644 --- a/.github/workflows/docker-test-cont.yml +++ b/.github/workflows/docker-test-cont.yml @@ -69,7 +69,6 @@ jobs: -e NODE_ENV=dev \ -e LOGGER_LEVEL=debug \ -e ALLOW_USER_HOTLINK_TEMPLATE=true \ - -e ALLOW_USER_SUPPLY_UNSAFE_DOMAIN=true \ -p 1200:1200 \ rsshub:latest diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index dd497b695422b9..01133479eb5b3f 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -83,7 +83,6 @@ jobs: run: pnpm start & env: ALLOW_USER_HOTLINK_TEMPLATE: true - ALLOW_USER_SUPPLY_UNSAFE_DOMAIN: true NODE_ENV: dev LOGGER_LEVEL: debug diff --git a/lib/config.ts b/lib/config.ts index 373af707bf0d72..228b4306ab9652 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -64,7 +64,6 @@ export type Config = { feature: { allow_user_hotlink_template: boolean; filter_regex_engine: string; - allow_user_supply_unsafe_domain: boolean; }; suffix?: string; titleLengthLimit: number; @@ -441,7 +440,6 @@ const calculateValue = () => { feature: { allow_user_hotlink_template: toBoolean(envs.ALLOW_USER_HOTLINK_TEMPLATE, false), filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', - allow_user_supply_unsafe_domain: toBoolean(envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN, false), }, suffix: envs.SUFFIX, titleLengthLimit: toInt(envs.TITLE_LENGTH_LIMIT, 150), diff --git a/lib/routes-deprecated/gitlab/common.js b/lib/routes-deprecated/gitlab/common.js deleted file mode 100644 index 5ad06ad4ebbae0..00000000000000 --- a/lib/routes-deprecated/gitlab/common.js +++ /dev/null @@ -1,5 +0,0 @@ -const allowHost = ['gitlab.com']; - -module.exports = { - allowHost, -}; diff --git a/lib/routes-deprecated/gitlab/explore.js b/lib/routes-deprecated/gitlab/explore.js index 0bb6ba43b312d9..09cfa072651531 100644 --- a/lib/routes-deprecated/gitlab/explore.js +++ b/lib/routes-deprecated/gitlab/explore.js @@ -1,7 +1,5 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const config = require('@/config').value; -const { allowHost } = require('./common'); module.exports = async (ctx) => { let { type, host } = ctx.params; @@ -12,9 +10,6 @@ module.exports = async (ctx) => { starred: 'Most stars', all: 'All', }; - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(new URL(`https://${host}/`).hostname)) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const res = await got({ method: 'get', diff --git a/lib/routes-deprecated/gitlab/release.js b/lib/routes-deprecated/gitlab/release.js index 094493e1cb45c8..3d35cda26f99cb 100644 --- a/lib/routes-deprecated/gitlab/release.js +++ b/lib/routes-deprecated/gitlab/release.js @@ -1,13 +1,8 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); -const config = require('@/config').value; -const { allowHost } = require('./common'); module.exports = async (ctx) => { const { namespace, project, host } = ctx.params; - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const host_ = host ?? 'gitlab.com'; const namespace_ = encodeURIComponent(namespace); diff --git a/lib/routes-deprecated/gitlab/tag.js b/lib/routes-deprecated/gitlab/tag.js index 68252e8fca1ae7..a3a5dd44ef4573 100644 --- a/lib/routes-deprecated/gitlab/tag.js +++ b/lib/routes-deprecated/gitlab/tag.js @@ -1,13 +1,8 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); -const config = require('@/config').value; -const { allowHost } = require('./common'); module.exports = async (ctx) => { const { namespace, project, host } = ctx.params; - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const host_ = host ?? 'gitlab.com'; const namespace_ = encodeURIComponent(namespace); diff --git a/lib/routes-deprecated/hexo/fluid.js b/lib/routes-deprecated/hexo/fluid.js index cae0672f6ea394..36c0892bf1ca06 100644 --- a/lib/routes-deprecated/hexo/fluid.js +++ b/lib/routes-deprecated/hexo/fluid.js @@ -3,9 +3,6 @@ const got = require('@/utils/got'); const config = require('@/config').value; module.exports = async (ctx) => { - if (!config.feature.allow_user_supply_unsafe_domain) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${ctx.params.url}`; const res = await got.get(`${url}/archives/`); const $ = cheerio.load(res.data); diff --git a/lib/routes-deprecated/hexo/next.js b/lib/routes-deprecated/hexo/next.js index 46c44887abc453..e2f3bebdb2b3e0 100644 --- a/lib/routes-deprecated/hexo/next.js +++ b/lib/routes-deprecated/hexo/next.js @@ -3,9 +3,6 @@ const got = require('@/utils/got'); const config = require('@/config').value; module.exports = async (ctx) => { - if (!config.feature.allow_user_supply_unsafe_domain) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${ctx.params.url}`; const res = await got.get(`${url}/archives/`); const $ = cheerio.load(res.data); diff --git a/lib/routes-deprecated/hexo/yilia.js b/lib/routes-deprecated/hexo/yilia.js index c55544f8e78570..65d04a23e5712d 100644 --- a/lib/routes-deprecated/hexo/yilia.js +++ b/lib/routes-deprecated/hexo/yilia.js @@ -3,9 +3,6 @@ const got = require('@/utils/got'); const config = require('@/config').value; module.exports = async (ctx) => { - if (!config.feature.allow_user_supply_unsafe_domain) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${ctx.params.url}`; const res = await got.get(url); const $ = cheerio.load(res.data); diff --git a/lib/routes.test.ts b/lib/routes.test.ts index d9cb64df8fe4ce..e8476885e0bd31 100644 --- a/lib/routes.test.ts +++ b/lib/routes.test.ts @@ -4,8 +4,6 @@ import Parser from 'rss-parser'; const parser = new Parser(); import { config } from '@/config'; -process.env.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN = 'true'; - const routes = { '/test/:id': '/test/1', }; @@ -16,10 +14,7 @@ if (process.env.FULL_ROUTES_TEST) { const requireConfig = namespaces[namespace].routes[route].features?.requireConfig; let configs; if (typeof requireConfig !== 'boolean') { - configs = requireConfig - ?.filter((config) => !config.optional) - .map((config) => config.name) - .filter((name) => name !== 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN'); + configs = requireConfig?.filter((config) => !config.optional).map((config) => config.name); } if (namespaces[namespace].routes[route].example && !configs?.length) { routes[`/${namespace}${route}`] = namespaces[namespace].routes[route].example; diff --git a/lib/routes/18comic/utils.ts b/lib/routes/18comic/utils.ts index e4ea27d1b3746a..2739d09fe68665 100644 --- a/lib/routes/18comic/utils.ts +++ b/lib/routes/18comic/utils.ts @@ -7,20 +7,10 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; const defaultDomain = 'jmcomic1.me'; -// list of address: https://jmcomic2.bet -const allowDomain = new Set(['18comic.vip', '18comic.org', 'jmcomic.me', 'jmcomic1.me', 'jm-comic3.art', 'jm-comic.club', 'jm-comic2.ark']); -const getRootUrl = (domain) => { - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(domain)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - - return `https://${domain}`; -}; +const getRootUrl = (domain) => `https://${domain}`; const ProcessItems = async (ctx, currentUrl, rootUrl) => { currentUrl = currentUrl.replace(/\?$/, ''); diff --git a/lib/routes/91porn/author.ts b/lib/routes/91porn/author.ts index fb6837f35fb180..548f719b99bd64 100644 --- a/lib/routes/91porn/author.ts +++ b/lib/routes/91porn/author.ts @@ -8,7 +8,6 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { domainValidation } from './utils'; export const route: Route = { path: '/author/:uid/:lang?', @@ -39,7 +38,6 @@ async function handler(ctx) { const { domain = '91porn.com' } = ctx.req.query(); const { uid, lang = 'en_US' } = ctx.req.param(); const siteUrl = `https://${domain}/uvideos.php?UID=${uid}&type=public`; - domainValidation(domain); const response = await got.post(siteUrl, { form: { diff --git a/lib/routes/91porn/index.ts b/lib/routes/91porn/index.ts index e704db6ae970e6..a1992b4d227257 100644 --- a/lib/routes/91porn/index.ts +++ b/lib/routes/91porn/index.ts @@ -8,7 +8,6 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { domainValidation } from './utils'; export const route: Route = { path: '/:lang?', @@ -42,7 +41,6 @@ async function handler(ctx) { const { domain = '91porn.com' } = ctx.req.query(); const siteUrl = `https://${domain}/index.php`; const { lang = 'en_US' } = ctx.req.param(); - domainValidation(domain); const response = await got.post(siteUrl, { form: { diff --git a/lib/routes/91porn/utils.ts b/lib/routes/91porn/utils.ts deleted file mode 100644 index 36c6a2e6157b01..00000000000000 --- a/lib/routes/91porn/utils.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowDomain = new Set(['91porn.com', 'www.91porn.com', '0122.91p30.com', 'www.91zuixindizhi.com', 'w1218.91p46.com']); - -const domainValidation = (domain) => { - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(domain)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } -}; - -export { domainValidation }; diff --git a/lib/routes/bdys/index.ts b/lib/routes/bdys/index.ts index 5c541d877cc8ab..661f56e9609321 100644 --- a/lib/routes/bdys/index.ts +++ b/lib/routes/bdys/index.ts @@ -10,11 +10,6 @@ import timezone from '@/utils/timezone'; import { art } from '@/utils/render'; import path from 'node:path'; import asyncPool from 'tiny-async-pool'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; - -// Visit https://www.bdys.me for the list of domains -const allowDomains = new Set(['52bdys.com', 'bde4.icu', 'bdys01.com']); export const route: Route = { path: '/:caty?/:type?/:area?/:year?/:order?', @@ -107,9 +102,6 @@ async function handler(ctx) { const order = ctx.req.param('order') || '0'; const site = ctx.req.query('domain') || 'bdys01.com'; - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomains.has(new URL(`https://${site}`).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const rootUrl = `https://www.${site}`; const currentUrl = `${rootUrl}/s/${caty}?${type === 'all' ? '' : '&type=' + type}${area === 'all' ? '' : '&area=' + area}${year === 'all' ? '' : '&year=' + year}&order=${order}`; diff --git a/lib/routes/biquge/index.ts b/lib/routes/biquge/index.ts index 581d94db85a465..2ae1e2d43249a0 100644 --- a/lib/routes/biquge/index.ts +++ b/lib/routes/biquge/index.ts @@ -6,25 +6,6 @@ import { load } from 'cheerio'; import iconv from 'iconv-lite'; import timezone from '@/utils/timezone'; import { parseDate } from '@/utils/parse-date'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowHost = new Set([ - 'www.xbiquwx.la', - 'www.biqu5200.net', - 'www.xbiquge.so', - 'www.biqugeu.net', - 'www.b520.cc', - 'www.ahfgb.com', - 'www.ibiquge.la', - 'www.biquge.tv', - 'www.bswtan.com', - 'www.biquge.co', - 'www.bqzhh.com', - 'www.biqugse.com', - 'www.ibiquge.info', - 'www.ishuquge.com', - 'www.mayiwxw.com', -]); export const route: Route = { path: '*', @@ -36,9 +17,6 @@ export const route: Route = { async function handler(ctx) { const rootUrl = getSubPath(ctx).split('/').slice(1, 4).join('/'); const currentUrl = getSubPath(ctx).slice(1); - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.has(new URL(rootUrl).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const response = await got({ method: 'get', diff --git a/lib/routes/btzj/index.ts b/lib/routes/btzj/index.ts index 58687e1a240098..a24a725f3b5e61 100644 --- a/lib/routes/btzj/index.ts +++ b/lib/routes/btzj/index.ts @@ -9,9 +9,6 @@ import timezone from '@/utils/timezone'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowDomain = new Set(['2btjia.com', '88btbtt.com', 'btbtt15.com', 'btbtt20.com']); export const route: Route = { path: '/:category?', @@ -71,9 +68,6 @@ export const route: Route = { async function handler(ctx) { let category = ctx.req.param('category') ?? ''; let domain = ctx.req.query('domain') ?? 'btbtt15.com'; - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(new URL(`http://${domain}/`).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } if (category === 'base') { category = ''; diff --git a/lib/routes/domp4/detail.ts b/lib/routes/domp4/detail.ts index ba9780237065a3..446b1a96f26bf9 100644 --- a/lib/routes/domp4/detail.ts +++ b/lib/routes/domp4/detail.ts @@ -2,7 +2,7 @@ import { Route } from '@/types'; import { load } from 'cheerio'; import got from '@/utils/got'; -import { decodeCipherText, composeMagnetUrl, getUrlType, ensureDomain } from './utils'; +import { decodeCipherText, composeMagnetUrl, getUrlType, defaultDomain } from './utils'; // 兼容没有 script 标签的情况，直接解析 dom function getDomList($, detailUrl) { @@ -111,7 +111,7 @@ async function handler(ctx) { if (/^\d+$/.test(pureId)) { detailType = 'detail'; } - const detailUrl = `${ensureDomain(ctx, domain)}/${detailType}/${pureId}.html`; + const detailUrl = `https://${domain ?? defaultDomain}/${detailType}/${pureId}.html`; const res = await got(detailUrl); const $ = load(res.data); diff --git a/lib/routes/domp4/latest-movie-bt.ts b/lib/routes/domp4/latest-movie-bt.ts index b9c38a011339bc..027faa48e30072 100644 --- a/lib/routes/domp4/latest-movie-bt.ts +++ b/lib/routes/domp4/latest-movie-bt.ts @@ -3,7 +3,7 @@ import got from '@/utils/got'; import { load } from 'cheerio'; import { getItemList as detailItemList } from './detail'; -import { defaultDomain, ensureDomain } from './utils'; +import { defaultDomain } from './utils'; import cache from '@/utils/cache'; function getItemList($) { @@ -47,7 +47,7 @@ export const route: Route = { async function handler(ctx) { const { domain, second } = ctx.req.query(); - const hostUrl = ensureDomain(ctx, domain); + const hostUrl = `https://${domain ?? defaultDomain}`; const latestUrl = `${hostUrl}/custom/update.html`; const res = await got.get(latestUrl); const $ = load(res.data); diff --git a/lib/routes/domp4/latest.ts b/lib/routes/domp4/latest.ts index c2e6705c749e82..0cf9ebcc036f5e 100644 --- a/lib/routes/domp4/latest.ts +++ b/lib/routes/domp4/latest.ts @@ -2,7 +2,7 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { defaultDomain, ensureDomain } from './utils'; +import { defaultDomain } from './utils'; function getItemList($, type) { const list = $(`#${type} .list-group-item`) @@ -45,7 +45,7 @@ async function handler(ctx) { const { type = 'vod' } = ctx.req.param(); const { domain } = ctx.req.query(); - const hostUrl = ensureDomain(ctx, domain); + const hostUrl = `https://${domain ?? defaultDomain}`; const latestUrl = `${hostUrl}/custom/update.html`; const res = await got.get(latestUrl); diff --git a/lib/routes/domp4/utils.ts b/lib/routes/domp4/utils.ts index 8c8c7b22c120ae..70602e10cdacf8 100644 --- a/lib/routes/domp4/utils.ts +++ b/lib/routes/domp4/utils.ts @@ -1,10 +1,5 @@ -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; - const defaultDomain = 'mp4us.com'; -const allowedDomains = new Set(['domp4.cc', 'mp4us.com', 'wemp4.com', 'dbmp4.com']); - /** * trackers from https://www.domp4.cc/Style/2020/js/base.js?v=2 */ @@ -85,12 +80,4 @@ function decodeCipherText(p, a, c, k, e, d) { return p; } -function ensureDomain(ctx, domain = defaultDomain) { - const origin = `https://${domain}`; - if (!config.feature.allow_user_supply_unsafe_domain && !allowedDomains.has(new URL(origin).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - return origin; -} - -export { defaultDomain, magnetTrackers, getUrlType, composeMagnetUrl, decodeCipherText, ensureDomain }; +export { defaultDomain, magnetTrackers, getUrlType, composeMagnetUrl, decodeCipherText }; diff --git a/lib/routes/fediverse/timeline.ts b/lib/routes/fediverse/timeline.ts index ed3e355fc2f675..fa1c74c1140aed 100644 --- a/lib/routes/fediverse/timeline.ts +++ b/lib/routes/fediverse/timeline.ts @@ -3,8 +3,6 @@ import { Route, ViewType } from '@/types'; import { parseDate } from '@/utils/parse-date'; import ofetch from '@/utils/ofetch'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/timeline/:account', @@ -25,7 +23,6 @@ export const route: Route = { handler, }; -const allowedDomain = new Set(['mastodon.social', 'pawoo.net', config.mastodon.apiHost].filter(Boolean)); const activityPubTypes = new Set(['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']); async function handler(ctx) { @@ -36,9 +33,6 @@ async function handler(ctx) { if (!domain || !username) { throw new InvalidParameterError('Invalid account'); } - if (!config.feature.allow_user_supply_unsafe_domain && !allowedDomain.has(domain.toLowerCase())) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const requestOptions = { headers: { diff --git a/lib/routes/javbus/index.ts b/lib/routes/javbus/index.ts index f07d9b3ebcfe11..9b93111768e00a 100644 --- a/lib/routes/javbus/index.ts +++ b/lib/routes/javbus/index.ts @@ -9,16 +9,12 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; const toSize = (raw) => { const matches = raw.match(/(\d+(\.\d+)?)(\w+)/); return matches[3] === 'GB' ? matches[1] * 1024 : matches[1]; }; -const allowDomain = new Set(['javbus.com', 'javbus.org', 'javsee.icu', 'javsee.one']); - export const route: Route = { path: '/:path{.+}?', radar: [ @@ -49,10 +45,6 @@ async function handler(ctx) { const rootUrl = `https://www.${domain}`; const westernUrl = `https://www.${westernDomain}`; - if (!config.feature.allow_user_supply_unsafe_domain && (!allowDomain.has(new URL(`https://${domain}/`).hostname) || !allowDomain.has(new URL(`https://${westernDomain}/`).hostname))) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - const currentUrl = `${isWestern ? westernUrl : rootUrl}${getSubPath(ctx) .replace(/^\/western/, '') .replace(/\/home/, '')}`; diff --git a/lib/routes/javdb/utils.ts b/lib/routes/javdb/utils.ts index fc4b0b6171badc..cadcb58bf7beb5 100644 --- a/lib/routes/javdb/utils.ts +++ b/lib/routes/javdb/utils.ts @@ -5,15 +5,9 @@ import { parseDate } from '@/utils/parse-date'; import { config } from '@/config'; import { Cookie, CookieJar } from 'tough-cookie'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowDomain = new Set(['javdb.com', 'javdb36.com', 'javdb007.com', 'javdb521.com']); - const ProcessItems = async (ctx, currentUrl, title) => { const domain = ctx.req.query('domain') ?? 'javdb.com'; const url = new URL(currentUrl, `https://${domain}`); - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(url.hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const rootUrl = `https://${domain}`; diff --git a/lib/routes/lemmy/index.ts b/lib/routes/lemmy/index.ts index cd2c240db50276..d4c277c0465b3e 100644 --- a/lib/routes/lemmy/index.ts +++ b/lib/routes/lemmy/index.ts @@ -6,7 +6,6 @@ import MarkdownIt from 'markdown-it'; const md = MarkdownIt({ html: true }); import { config } from '@/config'; import InvalidParameterError from '@/errors/types/invalid-parameter'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/:community/:sort?', @@ -41,12 +40,6 @@ export const route: Route = { }, }, features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: '', - }, - ], requirePuppeteer: false, antiCrawler: false, supportBT: false, @@ -66,10 +59,6 @@ async function handler(ctx) { throw new InvalidParameterError(`Invalid community: ${community}`); } const instance = community.split('@')[1]; - const allowedDomain = ['lemmy.world', 'lemm.ee', 'lemmy.ml', 'sh.itjust.works', 'feddit.de', 'hexbear.net', 'beehaw.org', 'lemmynsfw.com', 'lemmy.ca', 'programming.dev']; - if (!config.feature.allow_user_supply_unsafe_domain && !allowedDomain.includes(new URL(`http://${instance}/`).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const communityUrl = `https://${instance}/api/v3/community?name=${community}`; const communityData = await cache.tryGet(communityUrl, async () => { diff --git a/lib/routes/mastodon/account-id.ts b/lib/routes/mastodon/account-id.ts index f1fb49110e228f..38f30b552b67a4 100644 --- a/lib/routes/mastodon/account-id.ts +++ b/lib/routes/mastodon/account-id.ts @@ -1,7 +1,5 @@ import { Route, ViewType } from '@/types'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/account_id/:site/:account_id/statuses/:only_media?', @@ -37,9 +35,6 @@ async function handler(ctx) { const site = ctx.req.param('site'); const account_id = ctx.req.param('account_id'); const only_media = ctx.req.param('only_media') ? 'true' : 'false'; - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const { account_data, data } = await utils.getAccountStatuses(site, account_id, only_media); diff --git a/lib/routes/mastodon/timeline-local.ts b/lib/routes/mastodon/timeline-local.ts index 1713c9e8fba5bb..ef793f68d18c2c 100644 --- a/lib/routes/mastodon/timeline-local.ts +++ b/lib/routes/mastodon/timeline-local.ts @@ -1,8 +1,6 @@ import { Route, ViewType } from '@/types'; import got from '@/utils/got'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/timeline/:site/:only_media?', @@ -31,15 +29,11 @@ export const route: Route = { name: 'Instance timeline (local)', maintainers: ['hoilc'], handler, - description: `If the instance address is not \`mastodon.social\` or \`pawoo.net\`, then the route requires \`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN\` to be \`true\`.`, }; async function handler(ctx) { const site = ctx.req.param('site'); const only_media = ctx.req.param('only_media') ? 'true' : 'false'; - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${site}/api/v1/timelines/public?local=true&only_media=${only_media}`; diff --git a/lib/routes/mastodon/timeline-remote.ts b/lib/routes/mastodon/timeline-remote.ts index 5e962735371731..f6a387b4f97899 100644 --- a/lib/routes/mastodon/timeline-remote.ts +++ b/lib/routes/mastodon/timeline-remote.ts @@ -1,8 +1,6 @@ import { Route, ViewType } from '@/types'; import got from '@/utils/got'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/remote/:site/:only_media?', @@ -31,15 +29,11 @@ export const route: Route = { name: 'Instance timeline (federated)', maintainers: ['hoilc'], handler, - description: `If the instance address is not \`mastodon.social\` or \`pawoo.net\`, then the route requires \`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN\` to be \`true\`.`, }; async function handler(ctx) { const site = ctx.req.param('site'); const only_media = ctx.req.param('only_media') ? 'true' : 'false'; - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${site}/api/v1/timelines/public?remote=true&only_media=${only_media}`; diff --git a/lib/routes/mastodon/utils.ts b/lib/routes/mastodon/utils.ts index 40b4401ade10f5..668be328032557 100644 --- a/lib/routes/mastodon/utils.ts +++ b/lib/routes/mastodon/utils.ts @@ -96,9 +96,6 @@ async function getAccountIdByAcct(acct) { if (!(site && acctDomain)) { throw new ConfigNotFoundError('Mastodon RSS is disabled due to the lack of relevant config'); } - if (!config.feature.allow_user_supply_unsafe_domain && !allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`RSS for this domain is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true' or 'MASTODON_API_HOST' is set.`); - } const search_url = `https://${site}/api/v2/search`; const cacheUid = `mastodon_acct_id/${site}/${acct}`; diff --git a/lib/routes/misskey/featured-notes.ts b/lib/routes/misskey/featured-notes.ts index ac7751bba261c8..8712cf3a1e9c77 100644 --- a/lib/routes/misskey/featured-notes.ts +++ b/lib/routes/misskey/featured-notes.ts @@ -1,8 +1,6 @@ import { Route, ViewType } from '@/types'; import got from '@/utils/got'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/notes/featured/:site', @@ -25,9 +23,6 @@ export const route: Route = { async function handler(ctx) { const site = ctx.req.param('site'); - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } // docs on: https://misskey-hub.net/docs/api/endpoints/notes/featured.html const url = `https://${site}/api/notes/featured`; diff --git a/lib/routes/misskey/user-timeline.ts b/lib/routes/misskey/user-timeline.ts index 29fb2f35fc7825..c4ecb066156c53 100644 --- a/lib/routes/misskey/user-timeline.ts +++ b/lib/routes/misskey/user-timeline.ts @@ -1,7 +1,5 @@ import { Route, ViewType } from '@/types'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; import InvalidParameterError from '@/errors/types/invalid-parameter'; export const route: Route = { @@ -29,9 +27,6 @@ async function handler(ctx) { if (!pureUsername || !site) { throw new InvalidParameterError('Provide a valid Misskey username'); } - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const { accountData } = await utils.getUserTimelineByUsername(pureUsername, site); diff --git a/lib/routes/rsshub/transform/html.ts b/lib/routes/rsshub/transform/html.ts index d63c73782c24b1..bfaaee84ad461c 100644 --- a/lib/routes/rsshub/transform/html.ts +++ b/lib/routes/rsshub/transform/html.ts @@ -1,8 +1,6 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/transform/html/:url/:routeParams', @@ -10,12 +8,6 @@ export const route: Route = { example: '/rsshub/transform/html/https%3A%2F%2Fwechat2rss.xlab.app%2Fposts%2Flist%2F/item=div%5Bclass%3D%27post%2Dcontent%27%5D%20p%20a', parameters: { url: '`encodeURIComponent`ed URL address', routeParams: 'Transformation rules, requires URL encode' }, features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: '', - }, - ], requirePuppeteer: false, antiCrawler: false, supportBT: false, @@ -57,9 +49,6 @@ Specify options (in the format of query string) in parameter \`routeParams\` par }; async function handler(ctx) { - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = ctx.req.param('url'); const response = await got({ method: 'get', diff --git a/lib/routes/rsshub/transform/json.ts b/lib/routes/rsshub/transform/json.ts index ec4ba9601d8982..994c441161b68b 100644 --- a/lib/routes/rsshub/transform/json.ts +++ b/lib/routes/rsshub/transform/json.ts @@ -1,8 +1,6 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; function jsonGet(obj, attr) { if (typeof attr !== 'string') { @@ -22,12 +20,6 @@ export const route: Route = { example: '/rsshub/transform/json/https%3A%2F%2Fapi.github.com%2Frepos%2Fginuerzh%2Fgost%2Freleases/title=Gost%20releases&itemTitle=tag_name&itemLink=html_url&itemDesc=body', parameters: { url: '`encodeURIComponent`ed URL address', routeParams: 'Transformation rules, requires URL encode' }, features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: '', - }, - ], requirePuppeteer: false, antiCrawler: false, supportBT: false, @@ -70,9 +62,6 @@ JSON Path only supports format like \`a.b.c\`. if you need to access arrays, lik }; async function handler(ctx) { - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = ctx.req.param('url'); const response = await got({ method: 'get', diff --git a/lib/routes/rsshub/transform/sitemap.ts b/lib/routes/rsshub/transform/sitemap.ts index f013e943af0241..6994e03ef5eed0 100644 --- a/lib/routes/rsshub/transform/sitemap.ts +++ b/lib/routes/rsshub/transform/sitemap.ts @@ -1,8 +1,6 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/transform/sitemap/:url/:routeParams?', @@ -12,9 +10,6 @@ export const route: Route = { }; async function handler(ctx) { - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = ctx.req.param('url'); const response = await got({ method: 'get', diff --git a/lib/routes/wordpress/index.ts b/lib/routes/wordpress/index.ts index 05f0e4e0f41264..f085fa05eb238d 100644 --- a/lib/routes/wordpress/index.ts +++ b/lib/routes/wordpress/index.ts @@ -5,7 +5,6 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import parser from '@/utils/rss-parser'; import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; import { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr } from './util'; @@ -13,10 +12,6 @@ async function handler(ctx) { const { url = 'https://wordpress.org/news', filter } = ctx.req.param(); const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50; - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - if (!/^(https?):\/\/[^\s#$./?].\S*$/i.test(url)) { throw new Error('Invalid URL'); } @@ -148,19 +143,12 @@ export const route: Route = { You can also subscribe to multiple categories. \`/category/Podcast,Community\` to subscribe to both the Podcast and Community categories. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community). Categories and tags can be combined as well. \`/category/Releases/tag/tagging\` to subscribe to the Releases category and the tagging tag. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging). - + You can also search for keywords. \`/search/Blog\` to search for the keyword "Blog". In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog). :::`, categories: ['blog'], features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`, - optional: false, - }, - ], requirePuppeteer: false, antiCrawler: false, supportRadar: false, diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index 0ea437355eee41..5724f55e294cd4 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -1,6 +1,11 @@ import { createFetch } from 'ofetch'; import { config } from '@/config'; import logger from '@/utils/logger'; +import ip from 'ipaddr.js'; +import { lookup as nativeCallbackLookup } from 'dns'; +import { promisify } from 'util'; + +const lookup = promisify(nativeCallbackLookup); const rofetch = createFetch().create({ retryStatusCodes: [400, 408, 409, 425, 429, 500, 502, 503, 504], @@ -26,6 +31,20 @@ const rofetch = createFetch().create({ headers: { 'user-agent': config.ua, }, + onRequest: async ({ request }) => { + const url = typeof request === 'string' ? new URL(request) : new URL(request.url); + let addr; + if (ip.isValid(url.hostname)) { + addr = url.hostname; + } else { + const { address } = await lookup(url.hostname); + addr = address; + } + const parsedIp = ip.process(addr); + if (parsedIp.range() !== 'unicast') { + throw new Error('The IP of the domain is reserved!'); + } + }, }); export default rofetch; diff --git a/package.json b/package.json index ac11cc55386c82..a5e5c8717fe457 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "instagram-private-api": "1.46.1", "ioredis": "5.4.1", "ip-regex": "5.0.0", + "ipaddr.js": "^2.2.0", "jsdom": "25.0.1", "json-bigint": "1.0.0", "jsrsasign": "10.9.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4c1c76b9360b88..9c9783d67d8b75 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -134,6 +134,9 @@ importers: ip-regex: specifier: 5.0.0 version: 5.0.0 + ipaddr.js: + specifier: ^2.2.0 + version: 2.2.0 jsdom: specifier: 25.0.1 version: 25.0.1(bufferutil@4.0.8)(utf-8-validate@5.0.10) @@ -3657,6 +3660,10 @@ packages: resolution: {integrity: sha512-fOCG6lhoKKakwv+C6KdsOnGvgXnmgfmp0myi3bcNwj3qfwPAxRKWEuFhvEFF7ceYIz6+1jRZ+yguLFAmUNPEfw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + ipaddr.js@2.2.0: + resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + engines: {node: '>= 10'} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} @@ -9557,6 +9564,8 @@ snapshots: ip-regex@5.0.0: {} + ipaddr.js@2.2.0: {} + is-arrayish@0.2.1: {} is-arrayish@0.3.2: {} From 7fd4da603e1954d7452e038297396eebcbb9d795 Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:45:36 +0800 Subject: [PATCH 02/10] feat: ALLOW_CIDR whitelist --- lib/config.ts | 2 ++ lib/utils/ofetch.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/config.ts b/lib/config.ts index 228b4306ab9652..d8229abd5ee509 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -64,6 +64,7 @@ export type Config = { feature: { allow_user_hotlink_template: boolean; filter_regex_engine: string; + allow_cidr?: string; }; suffix?: string; titleLengthLimit: number; @@ -440,6 +441,7 @@ const calculateValue = () => { feature: { allow_user_hotlink_template: toBoolean(envs.ALLOW_USER_HOTLINK_TEMPLATE, false), filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', + allow_cidr: envs.ALLOW_CIDR, }, suffix: envs.SUFFIX, titleLengthLimit: toInt(envs.TITLE_LENGTH_LIMIT, 150), diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index 5724f55e294cd4..4dcbf975e8a4af 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -41,9 +41,16 @@ const rofetch = createFetch().create({ addr = address; } const parsedIp = ip.process(addr); - if (parsedIp.range() !== 'unicast') { - throw new Error('The IP of the domain is reserved!'); + if (parsedIp.range() === 'unicast') { + return; } + if (config.feature.allow_cidr) { + const allowCIDR = ip.parseCIDR(config.feature.allow_cidr); + if (parsedIp.match(allowCIDR)) { + return; + } + } + throw new Error('The IP of the domain is reserved!'); }, }); From 7441e82290dd3ee38c58b05c1a5cba1580d5b2ee Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:49:52 +0800 Subject: [PATCH 03/10] fix: config --- lib/routes-deprecated/hexo/fluid.js | 1 - lib/routes-deprecated/hexo/next.js | 1 - lib/routes-deprecated/hexo/yilia.js | 1 - 3 files changed, 3 deletions(-) diff --git a/lib/routes-deprecated/hexo/fluid.js b/lib/routes-deprecated/hexo/fluid.js index 36c0892bf1ca06..5dc1f458ad7b46 100644 --- a/lib/routes-deprecated/hexo/fluid.js +++ b/lib/routes-deprecated/hexo/fluid.js @@ -1,6 +1,5 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const config = require('@/config').value; module.exports = async (ctx) => { const url = `http://${ctx.params.url}`; diff --git a/lib/routes-deprecated/hexo/next.js b/lib/routes-deprecated/hexo/next.js index e2f3bebdb2b3e0..9422fc49f162b9 100644 --- a/lib/routes-deprecated/hexo/next.js +++ b/lib/routes-deprecated/hexo/next.js @@ -1,6 +1,5 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const config = require('@/config').value; module.exports = async (ctx) => { const url = `http://${ctx.params.url}`; diff --git a/lib/routes-deprecated/hexo/yilia.js b/lib/routes-deprecated/hexo/yilia.js index 65d04a23e5712d..54d72693091bde 100644 --- a/lib/routes-deprecated/hexo/yilia.js +++ b/lib/routes-deprecated/hexo/yilia.js @@ -1,6 +1,5 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const config = require('@/config').value; module.exports = async (ctx) => { const url = `http://${ctx.params.url}`; From a1aadee4d0ff5369bd7c0e4576ede4c02de935ab Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:55:29 +0800 Subject: [PATCH 04/10] fix: test --- .github/workflows/docker-test-cont.yml | 1 + .github/workflows/issue-command.yml | 1 + lib/routes.test.ts | 2 ++ 3 files changed, 4 insertions(+) diff --git a/.github/workflows/docker-test-cont.yml b/.github/workflows/docker-test-cont.yml index 47f83ac92f7064..24998dce9af8fa 100644 --- a/.github/workflows/docker-test-cont.yml +++ b/.github/workflows/docker-test-cont.yml @@ -68,6 +68,7 @@ jobs: --name rsshub \ -e NODE_ENV=dev \ -e LOGGER_LEVEL=debug \ + -e ALLOW_CIDR='127.0.0.1/32' -e ALLOW_USER_HOTLINK_TEMPLATE=true \ -p 1200:1200 \ rsshub:latest diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index 01133479eb5b3f..9fbabbb3ca8781 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -83,6 +83,7 @@ jobs: run: pnpm start & env: ALLOW_USER_HOTLINK_TEMPLATE: true + ALLOW_CIDR: '127.0.0.1/32' NODE_ENV: dev LOGGER_LEVEL: debug diff --git a/lib/routes.test.ts b/lib/routes.test.ts index e8476885e0bd31..02f623a547c9c9 100644 --- a/lib/routes.test.ts +++ b/lib/routes.test.ts @@ -4,6 +4,8 @@ import Parser from 'rss-parser'; const parser = new Parser(); import { config } from '@/config'; +process.env.ALLOW_CIDR = '127.0.0.1/32'; + const routes = { '/test/:id': '/test/1', }; From cf5f9684370a68d0bb0459aef06f57514d87a9de Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:56:41 +0800 Subject: [PATCH 05/10] fix: eslint --- lib/utils/ofetch.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index 4dcbf975e8a4af..8f5f527ac7e7bc 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -46,6 +46,7 @@ const rofetch = createFetch().create({ } if (config.feature.allow_cidr) { const allowCIDR = ip.parseCIDR(config.feature.allow_cidr); + // eslint-disable-next-line unicorn/prefer-regexp-test if (parsedIp.match(allowCIDR)) { return; } From 792fd5281391e37465e812135ad97fc60d0b4556 Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Wed, 25 Sep 2024 02:05:00 +0800 Subject: [PATCH 06/10] fix: test --- lib/middleware/parameter.test.ts | 1 + lib/utils/got.test.ts | 2 ++ lib/utils/ofetch.test.ts | 2 ++ lib/utils/request-rewriter.test.ts | 1 + 4 files changed, 6 insertions(+) diff --git a/lib/middleware/parameter.test.ts b/lib/middleware/parameter.test.ts index 57debada33ca16..201f6f43dce4f4 100644 --- a/lib/middleware/parameter.test.ts +++ b/lib/middleware/parameter.test.ts @@ -3,6 +3,7 @@ import Parser from 'rss-parser'; process.env.OPENAI_API_KEY = 'sk-1234567890'; process.env.OPENAI_API_ENDPOINT = 'https://api.openai.mock/v1'; +process.env.ALLOW_CIDR = '127.0.0.1/32'; vi.mock('@/utils/request-rewriter', () => ({ default: null })); const { config } = await import('@/config'); diff --git a/lib/utils/got.test.ts b/lib/utils/got.test.ts index 0fc54147837137..92d66badaf4d8e 100644 --- a/lib/utils/got.test.ts +++ b/lib/utils/got.test.ts @@ -4,6 +4,8 @@ import got from '@/utils/got'; import { config } from '@/config'; import { Cookie, CookieJar } from 'tough-cookie'; +process.env.ALLOW_CIDR = '127.0.0.1/32'; + describe('got', () => { it('headers', async () => { const { data } = await got('http://rsshub.test/headers'); diff --git a/lib/utils/ofetch.test.ts b/lib/utils/ofetch.test.ts index 8815ad2adc641d..29dfeaffce0d8b 100644 --- a/lib/utils/ofetch.test.ts +++ b/lib/utils/ofetch.test.ts @@ -3,6 +3,8 @@ import { http, HttpResponse } from 'msw'; import ofetch from '@/utils/ofetch'; import { config } from '@/config'; +process.env.ALLOW_CIDR = '127.0.0.1/32'; + describe('ofetch', () => { it('headers', async () => { const data = await ofetch('http://rsshub.test/headers'); diff --git a/lib/utils/request-rewriter.test.ts b/lib/utils/request-rewriter.test.ts index e274128f354b28..82ca9e0a585134 100644 --- a/lib/utils/request-rewriter.test.ts +++ b/lib/utils/request-rewriter.test.ts @@ -6,6 +6,7 @@ import http from 'node:http'; process.env.PROXY_URI = 'http://rsshub.proxy:2333/'; process.env.PROXY_AUTH = 'rsshubtest'; process.env.PROXY_URL_REGEX = 'headers'; +process.env.ALLOW_CIDR = '127.0.0.1/32'; await import('@/utils/request-rewriter'); const { config } = await import('@/config'); From 44b23adbe755a8d692f301dbfb26ca16f79a07c4 Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Sat, 12 Oct 2024 12:40:36 +0800 Subject: [PATCH 07/10] fix: enable timeout --- lib/utils/ofetch.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index 8f5f527ac7e7bc..37e0d8bf8cea9b 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -11,7 +11,7 @@ const rofetch = createFetch().create({ retryStatusCodes: [400, 408, 409, 425, 429, 500, 502, 503, 504], retry: config.requestRetry, retryDelay: 1000, - // timeout: config.requestTimeout, + timeout: config.requestTimeout, onResponseError({ request, response, options }) { if (options.retry) { logger.warn(`Request ${request} with error ${response.status} remaining retry attempts: ${options.retry}`); From e71051a9f8266831d39e8b7eef6c13b6c9f8b89e Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Sat, 12 Oct 2024 13:03:57 +0800 Subject: [PATCH 08/10] fix: allow test hostname --- lib/routes.test.ts | 2 -- lib/utils/got.test.ts | 2 -- lib/utils/ofetch.test.ts | 2 -- lib/utils/ofetch.ts | 16 +++++++++++----- lib/utils/request-rewriter.test.ts | 1 - 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/lib/routes.test.ts b/lib/routes.test.ts index 02f623a547c9c9..e8476885e0bd31 100644 --- a/lib/routes.test.ts +++ b/lib/routes.test.ts @@ -4,8 +4,6 @@ import Parser from 'rss-parser'; const parser = new Parser(); import { config } from '@/config'; -process.env.ALLOW_CIDR = '127.0.0.1/32'; - const routes = { '/test/:id': '/test/1', }; diff --git a/lib/utils/got.test.ts b/lib/utils/got.test.ts index 92d66badaf4d8e..0fc54147837137 100644 --- a/lib/utils/got.test.ts +++ b/lib/utils/got.test.ts @@ -4,8 +4,6 @@ import got from '@/utils/got'; import { config } from '@/config'; import { Cookie, CookieJar } from 'tough-cookie'; -process.env.ALLOW_CIDR = '127.0.0.1/32'; - describe('got', () => { it('headers', async () => { const { data } = await got('http://rsshub.test/headers'); diff --git a/lib/utils/ofetch.test.ts b/lib/utils/ofetch.test.ts index 29dfeaffce0d8b..8815ad2adc641d 100644 --- a/lib/utils/ofetch.test.ts +++ b/lib/utils/ofetch.test.ts @@ -3,8 +3,6 @@ import { http, HttpResponse } from 'msw'; import ofetch from '@/utils/ofetch'; import { config } from '@/config'; -process.env.ALLOW_CIDR = '127.0.0.1/32'; - describe('ofetch', () => { it('headers', async () => { const data = await ofetch('http://rsshub.test/headers'); diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index 37e0d8bf8cea9b..8ec9b5c801a938 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -34,11 +34,17 @@ const rofetch = createFetch().create({ onRequest: async ({ request }) => { const url = typeof request === 'string' ? new URL(request) : new URL(request.url); let addr; - if (ip.isValid(url.hostname)) { - addr = url.hostname; - } else { - const { address } = await lookup(url.hostname); - addr = address; + // for test + if (url.hostname.endsWith('.test') || url.hostname.endsWith('.mock') || url.hostname.endsWith('.proxy')) { + return; + } + { + if (ip.isValid(url.hostname)) { + addr = url.hostname; + } else { + const { address } = await lookup(url.hostname); + addr = address; + } } const parsedIp = ip.process(addr); if (parsedIp.range() === 'unicast') { diff --git a/lib/utils/request-rewriter.test.ts b/lib/utils/request-rewriter.test.ts index 82ca9e0a585134..e274128f354b28 100644 --- a/lib/utils/request-rewriter.test.ts +++ b/lib/utils/request-rewriter.test.ts @@ -6,7 +6,6 @@ import http from 'node:http'; process.env.PROXY_URI = 'http://rsshub.proxy:2333/'; process.env.PROXY_AUTH = 'rsshubtest'; process.env.PROXY_URL_REGEX = 'headers'; -process.env.ALLOW_CIDR = '127.0.0.1/32'; await import('@/utils/request-rewriter'); const { config } = await import('@/config'); From 37ac0e11777219c368f4f0f2397862a6b48e3531 Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Sat, 12 Oct 2024 13:07:49 +0800 Subject: [PATCH 09/10] fix: allow test hostname --- lib/middleware/parameter.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/middleware/parameter.test.ts b/lib/middleware/parameter.test.ts index 201f6f43dce4f4..57debada33ca16 100644 --- a/lib/middleware/parameter.test.ts +++ b/lib/middleware/parameter.test.ts @@ -3,7 +3,6 @@ import Parser from 'rss-parser'; process.env.OPENAI_API_KEY = 'sk-1234567890'; process.env.OPENAI_API_ENDPOINT = 'https://api.openai.mock/v1'; -process.env.ALLOW_CIDR = '127.0.0.1/32'; vi.mock('@/utils/request-rewriter', () => ({ default: null })); const { config } = await import('@/config'); From 35678caa8baa844251a2fb597d901022a7a801a9 Mon Sep 17 00:00:00 2001 From: ttttmr <32825326+ttttmr@users.noreply.github.com> Date: Sat, 12 Oct 2024 13:32:12 +0800 Subject: [PATCH 10/10] fix: add test --- lib/utils/ofetch.test.ts | 16 ++++++++++++++++ lib/utils/ofetch.ts | 12 +++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/lib/utils/ofetch.test.ts b/lib/utils/ofetch.test.ts index 8815ad2adc641d..2dd29dedeeaf4c 100644 --- a/lib/utils/ofetch.test.ts +++ b/lib/utils/ofetch.test.ts @@ -48,4 +48,20 @@ describe('ofetch', () => { }); expect(response.test).toBe('rsshub'); }); + + it('ban-ip', async () => { + try { + await ofetch('http://192.168.1.1'); + } catch (error: any) { + expect(error.message).toContain('The IP of the domain is reserved!'); + } + }); + + it('ban-ip-domain', async () => { + try { + await ofetch('http://10.0.0.1.nip.io'); + } catch (error: any) { + expect(error.message).toContain('The IP of the domain is reserved!'); + } + }); }); diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index 8ec9b5c801a938..3b0e8ec5a3f19d 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -38,13 +38,11 @@ const rofetch = createFetch().create({ if (url.hostname.endsWith('.test') || url.hostname.endsWith('.mock') || url.hostname.endsWith('.proxy')) { return; } - { - if (ip.isValid(url.hostname)) { - addr = url.hostname; - } else { - const { address } = await lookup(url.hostname); - addr = address; - } + if (ip.isValid(url.hostname)) { + addr = url.hostname; + } else { + const { address } = await lookup(url.hostname); + addr = address; } const parsedIp = ip.process(addr); if (parsedIp.range() === 'unicast') {