diff --git a/.github/workflows/docker-test-cont.yml b/.github/workflows/docker-test-cont.yml index 761706042c0d47..24998dce9af8fa 100644 --- a/.github/workflows/docker-test-cont.yml +++ b/.github/workflows/docker-test-cont.yml @@ -68,8 +68,8 @@ jobs: --name rsshub \ -e NODE_ENV=dev \ -e LOGGER_LEVEL=debug \ + -e ALLOW_CIDR='127.0.0.1/32' -e ALLOW_USER_HOTLINK_TEMPLATE=true \ - -e ALLOW_USER_SUPPLY_UNSAFE_DOMAIN=true \ -p 1200:1200 \ rsshub:latest diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index dd497b695422b9..9fbabbb3ca8781 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -83,7 +83,7 @@ jobs: run: pnpm start & env: ALLOW_USER_HOTLINK_TEMPLATE: true - ALLOW_USER_SUPPLY_UNSAFE_DOMAIN: true + ALLOW_CIDR: '127.0.0.1/32' NODE_ENV: dev LOGGER_LEVEL: debug diff --git a/lib/config.ts b/lib/config.ts index e2e0e162f8794f..92955f34a3c48f 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -64,7 +64,7 @@ export type Config = { feature: { allow_user_hotlink_template: boolean; filter_regex_engine: string; - allow_user_supply_unsafe_domain: boolean; + allow_cidr?: string; }; suffix?: string; titleLengthLimit: number; @@ -450,7 +450,7 @@ const calculateValue = () => { feature: { allow_user_hotlink_template: toBoolean(envs.ALLOW_USER_HOTLINK_TEMPLATE, false), filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', - allow_user_supply_unsafe_domain: toBoolean(envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN, false), + allow_cidr: envs.ALLOW_CIDR, }, suffix: envs.SUFFIX, titleLengthLimit: toInt(envs.TITLE_LENGTH_LIMIT, 150), diff --git a/lib/routes-deprecated/gitlab/common.js b/lib/routes-deprecated/gitlab/common.js deleted file mode 100644 index 5ad06ad4ebbae0..00000000000000 --- a/lib/routes-deprecated/gitlab/common.js +++ /dev/null @@ -1,5 +0,0 @@ -const allowHost = ['gitlab.com']; - -module.exports = { - allowHost, -}; diff --git a/lib/routes-deprecated/gitlab/explore.js b/lib/routes-deprecated/gitlab/explore.js index 0bb6ba43b312d9..09cfa072651531 100644 --- a/lib/routes-deprecated/gitlab/explore.js +++ b/lib/routes-deprecated/gitlab/explore.js @@ -1,7 +1,5 @@ const got = require('@/utils/got'); const cheerio = require('cheerio'); -const config = require('@/config').value; -const { allowHost } = require('./common'); module.exports = async (ctx) => { let { type, host } = ctx.params; @@ -12,9 +10,6 @@ module.exports = async (ctx) => { starred: 'Most stars', all: 'All', }; - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(new URL(`https://${host}/`).hostname)) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const res = await got({ method: 'get', diff --git a/lib/routes-deprecated/gitlab/release.js b/lib/routes-deprecated/gitlab/release.js index 094493e1cb45c8..3d35cda26f99cb 100644 --- a/lib/routes-deprecated/gitlab/release.js +++ b/lib/routes-deprecated/gitlab/release.js @@ -1,13 +1,8 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); -const config = require('@/config').value; -const { allowHost } = require('./common'); module.exports = async (ctx) => { const { namespace, project, host } = ctx.params; - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const host_ = host ?? 'gitlab.com'; const namespace_ = encodeURIComponent(namespace); diff --git a/lib/routes-deprecated/gitlab/tag.js b/lib/routes-deprecated/gitlab/tag.js index 68252e8fca1ae7..a3a5dd44ef4573 100644 --- a/lib/routes-deprecated/gitlab/tag.js +++ b/lib/routes-deprecated/gitlab/tag.js @@ -1,13 +1,8 @@ const got = require('@/utils/got'); const { parseDate } = require('@/utils/parse-date'); -const config = require('@/config').value; -const { allowHost } = require('./common'); module.exports = async (ctx) => { const { namespace, project, host } = ctx.params; - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(host)) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const host_ = host ?? 'gitlab.com'; const namespace_ = encodeURIComponent(namespace); diff --git a/lib/routes-deprecated/hexo/fluid.js b/lib/routes-deprecated/hexo/fluid.js index cae0672f6ea394..5dc1f458ad7b46 100644 --- a/lib/routes-deprecated/hexo/fluid.js +++ b/lib/routes-deprecated/hexo/fluid.js @@ -1,11 +1,7 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const config = require('@/config').value; module.exports = async (ctx) => { - if (!config.feature.allow_user_supply_unsafe_domain) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${ctx.params.url}`; const res = await got.get(`${url}/archives/`); const $ = cheerio.load(res.data); diff --git a/lib/routes-deprecated/hexo/next.js b/lib/routes-deprecated/hexo/next.js index 46c44887abc453..9422fc49f162b9 100644 --- a/lib/routes-deprecated/hexo/next.js +++ b/lib/routes-deprecated/hexo/next.js @@ -1,11 +1,7 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const config = require('@/config').value; module.exports = async (ctx) => { - if (!config.feature.allow_user_supply_unsafe_domain) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${ctx.params.url}`; const res = await got.get(`${url}/archives/`); const $ = cheerio.load(res.data); diff --git a/lib/routes-deprecated/hexo/yilia.js b/lib/routes-deprecated/hexo/yilia.js index c55544f8e78570..54d72693091bde 100644 --- a/lib/routes-deprecated/hexo/yilia.js +++ b/lib/routes-deprecated/hexo/yilia.js @@ -1,11 +1,7 @@ const cheerio = require('cheerio'); const got = require('@/utils/got'); -const config = require('@/config').value; module.exports = async (ctx) => { - if (!config.feature.allow_user_supply_unsafe_domain) { - ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${ctx.params.url}`; const res = await got.get(url); const $ = cheerio.load(res.data); diff --git a/lib/routes.test.ts b/lib/routes.test.ts index d9cb64df8fe4ce..e8476885e0bd31 100644 --- a/lib/routes.test.ts +++ b/lib/routes.test.ts @@ -4,8 +4,6 @@ import Parser from 'rss-parser'; const parser = new Parser(); import { config } from '@/config'; -process.env.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN = 'true'; - const routes = { '/test/:id': '/test/1', }; @@ -16,10 +14,7 @@ if (process.env.FULL_ROUTES_TEST) { const requireConfig = namespaces[namespace].routes[route].features?.requireConfig; let configs; if (typeof requireConfig !== 'boolean') { - configs = requireConfig - ?.filter((config) => !config.optional) - .map((config) => config.name) - .filter((name) => name !== 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN'); + configs = requireConfig?.filter((config) => !config.optional).map((config) => config.name); } if (namespaces[namespace].routes[route].example && !configs?.length) { routes[`/${namespace}${route}`] = namespaces[namespace].routes[route].example; diff --git a/lib/routes/18comic/utils.ts b/lib/routes/18comic/utils.ts index e4ea27d1b3746a..2739d09fe68665 100644 --- a/lib/routes/18comic/utils.ts +++ b/lib/routes/18comic/utils.ts @@ -7,20 +7,10 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; const defaultDomain = 'jmcomic1.me'; -// list of address: https://jmcomic2.bet -const allowDomain = new Set(['18comic.vip', '18comic.org', 'jmcomic.me', 'jmcomic1.me', 'jm-comic3.art', 'jm-comic.club', 'jm-comic2.ark']); -const getRootUrl = (domain) => { - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(domain)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - - return `https://${domain}`; -}; +const getRootUrl = (domain) => `https://${domain}`; const ProcessItems = async (ctx, currentUrl, rootUrl) => { currentUrl = currentUrl.replace(/\?$/, ''); diff --git a/lib/routes/91porn/author.ts b/lib/routes/91porn/author.ts index fb6837f35fb180..548f719b99bd64 100644 --- a/lib/routes/91porn/author.ts +++ b/lib/routes/91porn/author.ts @@ -8,7 +8,6 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { domainValidation } from './utils'; export const route: Route = { path: '/author/:uid/:lang?', @@ -39,7 +38,6 @@ async function handler(ctx) { const { domain = '91porn.com' } = ctx.req.query(); const { uid, lang = 'en_US' } = ctx.req.param(); const siteUrl = `https://${domain}/uvideos.php?UID=${uid}&type=public`; - domainValidation(domain); const response = await got.post(siteUrl, { form: { diff --git a/lib/routes/91porn/index.ts b/lib/routes/91porn/index.ts index e704db6ae970e6..a1992b4d227257 100644 --- a/lib/routes/91porn/index.ts +++ b/lib/routes/91porn/index.ts @@ -8,7 +8,6 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { domainValidation } from './utils'; export const route: Route = { path: '/:lang?', @@ -42,7 +41,6 @@ async function handler(ctx) { const { domain = '91porn.com' } = ctx.req.query(); const siteUrl = `https://${domain}/index.php`; const { lang = 'en_US' } = ctx.req.param(); - domainValidation(domain); const response = await got.post(siteUrl, { form: { diff --git a/lib/routes/91porn/utils.ts b/lib/routes/91porn/utils.ts deleted file mode 100644 index 36c6a2e6157b01..00000000000000 --- a/lib/routes/91porn/utils.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowDomain = new Set(['91porn.com', 'www.91porn.com', '0122.91p30.com', 'www.91zuixindizhi.com', 'w1218.91p46.com']); - -const domainValidation = (domain) => { - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(domain)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } -}; - -export { domainValidation }; diff --git a/lib/routes/bdys/index.ts b/lib/routes/bdys/index.ts index 5c541d877cc8ab..661f56e9609321 100644 --- a/lib/routes/bdys/index.ts +++ b/lib/routes/bdys/index.ts @@ -10,11 +10,6 @@ import timezone from '@/utils/timezone'; import { art } from '@/utils/render'; import path from 'node:path'; import asyncPool from 'tiny-async-pool'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; - -// Visit https://www.bdys.me for the list of domains -const allowDomains = new Set(['52bdys.com', 'bde4.icu', 'bdys01.com']); export const route: Route = { path: '/:caty?/:type?/:area?/:year?/:order?', @@ -107,9 +102,6 @@ async function handler(ctx) { const order = ctx.req.param('order') || '0'; const site = ctx.req.query('domain') || 'bdys01.com'; - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomains.has(new URL(`https://${site}`).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const rootUrl = `https://www.${site}`; const currentUrl = `${rootUrl}/s/${caty}?${type === 'all' ? '' : '&type=' + type}${area === 'all' ? '' : '&area=' + area}${year === 'all' ? '' : '&year=' + year}&order=${order}`; diff --git a/lib/routes/biquge/index.ts b/lib/routes/biquge/index.ts index 581d94db85a465..2ae1e2d43249a0 100644 --- a/lib/routes/biquge/index.ts +++ b/lib/routes/biquge/index.ts @@ -6,25 +6,6 @@ import { load } from 'cheerio'; import iconv from 'iconv-lite'; import timezone from '@/utils/timezone'; import { parseDate } from '@/utils/parse-date'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowHost = new Set([ - 'www.xbiquwx.la', - 'www.biqu5200.net', - 'www.xbiquge.so', - 'www.biqugeu.net', - 'www.b520.cc', - 'www.ahfgb.com', - 'www.ibiquge.la', - 'www.biquge.tv', - 'www.bswtan.com', - 'www.biquge.co', - 'www.bqzhh.com', - 'www.biqugse.com', - 'www.ibiquge.info', - 'www.ishuquge.com', - 'www.mayiwxw.com', -]); export const route: Route = { path: '*', @@ -36,9 +17,6 @@ export const route: Route = { async function handler(ctx) { const rootUrl = getSubPath(ctx).split('/').slice(1, 4).join('/'); const currentUrl = getSubPath(ctx).slice(1); - if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.has(new URL(rootUrl).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const response = await got({ method: 'get', diff --git a/lib/routes/btzj/index.ts b/lib/routes/btzj/index.ts index 58687e1a240098..a24a725f3b5e61 100644 --- a/lib/routes/btzj/index.ts +++ b/lib/routes/btzj/index.ts @@ -9,9 +9,6 @@ import timezone from '@/utils/timezone'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowDomain = new Set(['2btjia.com', '88btbtt.com', 'btbtt15.com', 'btbtt20.com']); export const route: Route = { path: '/:category?', @@ -71,9 +68,6 @@ export const route: Route = { async function handler(ctx) { let category = ctx.req.param('category') ?? ''; let domain = ctx.req.query('domain') ?? 'btbtt15.com'; - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(new URL(`http://${domain}/`).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } if (category === 'base') { category = ''; diff --git a/lib/routes/domp4/detail.ts b/lib/routes/domp4/detail.ts index ba9780237065a3..446b1a96f26bf9 100644 --- a/lib/routes/domp4/detail.ts +++ b/lib/routes/domp4/detail.ts @@ -2,7 +2,7 @@ import { Route } from '@/types'; import { load } from 'cheerio'; import got from '@/utils/got'; -import { decodeCipherText, composeMagnetUrl, getUrlType, ensureDomain } from './utils'; +import { decodeCipherText, composeMagnetUrl, getUrlType, defaultDomain } from './utils'; // 兼容没有 script 标签的情况，直接解析 dom function getDomList($, detailUrl) { @@ -111,7 +111,7 @@ async function handler(ctx) { if (/^\d+$/.test(pureId)) { detailType = 'detail'; } - const detailUrl = `${ensureDomain(ctx, domain)}/${detailType}/${pureId}.html`; + const detailUrl = `https://${domain ?? defaultDomain}/${detailType}/${pureId}.html`; const res = await got(detailUrl); const $ = load(res.data); diff --git a/lib/routes/domp4/latest-movie-bt.ts b/lib/routes/domp4/latest-movie-bt.ts index b9c38a011339bc..027faa48e30072 100644 --- a/lib/routes/domp4/latest-movie-bt.ts +++ b/lib/routes/domp4/latest-movie-bt.ts @@ -3,7 +3,7 @@ import got from '@/utils/got'; import { load } from 'cheerio'; import { getItemList as detailItemList } from './detail'; -import { defaultDomain, ensureDomain } from './utils'; +import { defaultDomain } from './utils'; import cache from '@/utils/cache'; function getItemList($) { @@ -47,7 +47,7 @@ export const route: Route = { async function handler(ctx) { const { domain, second } = ctx.req.query(); - const hostUrl = ensureDomain(ctx, domain); + const hostUrl = `https://${domain ?? defaultDomain}`; const latestUrl = `${hostUrl}/custom/update.html`; const res = await got.get(latestUrl); const $ = load(res.data); diff --git a/lib/routes/domp4/latest.ts b/lib/routes/domp4/latest.ts index c2e6705c749e82..0cf9ebcc036f5e 100644 --- a/lib/routes/domp4/latest.ts +++ b/lib/routes/domp4/latest.ts @@ -2,7 +2,7 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { defaultDomain, ensureDomain } from './utils'; +import { defaultDomain } from './utils'; function getItemList($, type) { const list = $(`#${type} .list-group-item`) @@ -45,7 +45,7 @@ async function handler(ctx) { const { type = 'vod' } = ctx.req.param(); const { domain } = ctx.req.query(); - const hostUrl = ensureDomain(ctx, domain); + const hostUrl = `https://${domain ?? defaultDomain}`; const latestUrl = `${hostUrl}/custom/update.html`; const res = await got.get(latestUrl); diff --git a/lib/routes/domp4/utils.ts b/lib/routes/domp4/utils.ts index 8c8c7b22c120ae..70602e10cdacf8 100644 --- a/lib/routes/domp4/utils.ts +++ b/lib/routes/domp4/utils.ts @@ -1,10 +1,5 @@ -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; - const defaultDomain = 'mp4us.com'; -const allowedDomains = new Set(['domp4.cc', 'mp4us.com', 'wemp4.com', 'dbmp4.com']); - /** * trackers from https://www.domp4.cc/Style/2020/js/base.js?v=2 */ @@ -85,12 +80,4 @@ function decodeCipherText(p, a, c, k, e, d) { return p; } -function ensureDomain(ctx, domain = defaultDomain) { - const origin = `https://${domain}`; - if (!config.feature.allow_user_supply_unsafe_domain && !allowedDomains.has(new URL(origin).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - return origin; -} - -export { defaultDomain, magnetTrackers, getUrlType, composeMagnetUrl, decodeCipherText, ensureDomain }; +export { defaultDomain, magnetTrackers, getUrlType, composeMagnetUrl, decodeCipherText }; diff --git a/lib/routes/fediverse/timeline.ts b/lib/routes/fediverse/timeline.ts index dc199a2d8c48f9..5b55e768833e9a 100644 --- a/lib/routes/fediverse/timeline.ts +++ b/lib/routes/fediverse/timeline.ts @@ -3,8 +3,6 @@ import { Route, ViewType } from '@/types'; import { parseDate } from '@/utils/parse-date'; import ofetch from '@/utils/ofetch'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/timeline/:account', @@ -25,7 +23,6 @@ export const route: Route = { handler, }; -const allowedDomain = new Set(['mastodon.social', 'pawoo.net', config.mastodon.apiHost].filter(Boolean)); const activityPubTypes = new Set(['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"']); async function handler(ctx) { @@ -36,9 +33,6 @@ async function handler(ctx) { if (!domain || !username) { throw new InvalidParameterError('Invalid account'); } - if (!config.feature.allow_user_supply_unsafe_domain && !allowedDomain.has(domain.toLowerCase())) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const requestOptions = { headers: { diff --git a/lib/routes/javbus/index.ts b/lib/routes/javbus/index.ts index f07d9b3ebcfe11..9b93111768e00a 100644 --- a/lib/routes/javbus/index.ts +++ b/lib/routes/javbus/index.ts @@ -9,16 +9,12 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import { art } from '@/utils/render'; import path from 'node:path'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; const toSize = (raw) => { const matches = raw.match(/(\d+(\.\d+)?)(\w+)/); return matches[3] === 'GB' ? matches[1] * 1024 : matches[1]; }; -const allowDomain = new Set(['javbus.com', 'javbus.org', 'javsee.icu', 'javsee.one']); - export const route: Route = { path: '/:path{.+}?', radar: [ @@ -49,10 +45,6 @@ async function handler(ctx) { const rootUrl = `https://www.${domain}`; const westernUrl = `https://www.${westernDomain}`; - if (!config.feature.allow_user_supply_unsafe_domain && (!allowDomain.has(new URL(`https://${domain}/`).hostname) || !allowDomain.has(new URL(`https://${westernDomain}/`).hostname))) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - const currentUrl = `${isWestern ? westernUrl : rootUrl}${getSubPath(ctx) .replace(/^\/western/, '') .replace(/\/home/, '')}`; diff --git a/lib/routes/javdb/utils.ts b/lib/routes/javdb/utils.ts index fc4b0b6171badc..cadcb58bf7beb5 100644 --- a/lib/routes/javdb/utils.ts +++ b/lib/routes/javdb/utils.ts @@ -5,15 +5,9 @@ import { parseDate } from '@/utils/parse-date'; import { config } from '@/config'; import { Cookie, CookieJar } from 'tough-cookie'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; -const allowDomain = new Set(['javdb.com', 'javdb36.com', 'javdb007.com', 'javdb521.com']); - const ProcessItems = async (ctx, currentUrl, title) => { const domain = ctx.req.query('domain') ?? 'javdb.com'; const url = new URL(currentUrl, `https://${domain}`); - if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(url.hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const rootUrl = `https://${domain}`; diff --git a/lib/routes/lemmy/index.ts b/lib/routes/lemmy/index.ts index cd2c240db50276..d4c277c0465b3e 100644 --- a/lib/routes/lemmy/index.ts +++ b/lib/routes/lemmy/index.ts @@ -6,7 +6,6 @@ import MarkdownIt from 'markdown-it'; const md = MarkdownIt({ html: true }); import { config } from '@/config'; import InvalidParameterError from '@/errors/types/invalid-parameter'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/:community/:sort?', @@ -41,12 +40,6 @@ export const route: Route = { }, }, features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: '', - }, - ], requirePuppeteer: false, antiCrawler: false, supportBT: false, @@ -66,10 +59,6 @@ async function handler(ctx) { throw new InvalidParameterError(`Invalid community: ${community}`); } const instance = community.split('@')[1]; - const allowedDomain = ['lemmy.world', 'lemm.ee', 'lemmy.ml', 'sh.itjust.works', 'feddit.de', 'hexbear.net', 'beehaw.org', 'lemmynsfw.com', 'lemmy.ca', 'programming.dev']; - if (!config.feature.allow_user_supply_unsafe_domain && !allowedDomain.includes(new URL(`http://${instance}/`).hostname)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const communityUrl = `https://${instance}/api/v3/community?name=${community}`; const communityData = await cache.tryGet(communityUrl, async () => { diff --git a/lib/routes/mastodon/account-id.ts b/lib/routes/mastodon/account-id.ts index f1fb49110e228f..38f30b552b67a4 100644 --- a/lib/routes/mastodon/account-id.ts +++ b/lib/routes/mastodon/account-id.ts @@ -1,7 +1,5 @@ import { Route, ViewType } from '@/types'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/account_id/:site/:account_id/statuses/:only_media?', @@ -37,9 +35,6 @@ async function handler(ctx) { const site = ctx.req.param('site'); const account_id = ctx.req.param('account_id'); const only_media = ctx.req.param('only_media') ? 'true' : 'false'; - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const { account_data, data } = await utils.getAccountStatuses(site, account_id, only_media); diff --git a/lib/routes/mastodon/timeline-local.ts b/lib/routes/mastodon/timeline-local.ts index 1713c9e8fba5bb..ef793f68d18c2c 100644 --- a/lib/routes/mastodon/timeline-local.ts +++ b/lib/routes/mastodon/timeline-local.ts @@ -1,8 +1,6 @@ import { Route, ViewType } from '@/types'; import got from '@/utils/got'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/timeline/:site/:only_media?', @@ -31,15 +29,11 @@ export const route: Route = { name: 'Instance timeline (local)', maintainers: ['hoilc'], handler, - description: `If the instance address is not \`mastodon.social\` or \`pawoo.net\`, then the route requires \`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN\` to be \`true\`.`, }; async function handler(ctx) { const site = ctx.req.param('site'); const only_media = ctx.req.param('only_media') ? 'true' : 'false'; - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${site}/api/v1/timelines/public?local=true&only_media=${only_media}`; diff --git a/lib/routes/mastodon/timeline-remote.ts b/lib/routes/mastodon/timeline-remote.ts index 5e962735371731..f6a387b4f97899 100644 --- a/lib/routes/mastodon/timeline-remote.ts +++ b/lib/routes/mastodon/timeline-remote.ts @@ -1,8 +1,6 @@ import { Route, ViewType } from '@/types'; import got from '@/utils/got'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/remote/:site/:only_media?', @@ -31,15 +29,11 @@ export const route: Route = { name: 'Instance timeline (federated)', maintainers: ['hoilc'], handler, - description: `If the instance address is not \`mastodon.social\` or \`pawoo.net\`, then the route requires \`ALLOW_USER_SUPPLY_UNSAFE_DOMAIN\` to be \`true\`.`, }; async function handler(ctx) { const site = ctx.req.param('site'); const only_media = ctx.req.param('only_media') ? 'true' : 'false'; - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = `http://${site}/api/v1/timelines/public?remote=true&only_media=${only_media}`; diff --git a/lib/routes/mastodon/utils.ts b/lib/routes/mastodon/utils.ts index 40b4401ade10f5..668be328032557 100644 --- a/lib/routes/mastodon/utils.ts +++ b/lib/routes/mastodon/utils.ts @@ -96,9 +96,6 @@ async function getAccountIdByAcct(acct) { if (!(site && acctDomain)) { throw new ConfigNotFoundError('Mastodon RSS is disabled due to the lack of relevant config'); } - if (!config.feature.allow_user_supply_unsafe_domain && !allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`RSS for this domain is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true' or 'MASTODON_API_HOST' is set.`); - } const search_url = `https://${site}/api/v2/search`; const cacheUid = `mastodon_acct_id/${site}/${acct}`; diff --git a/lib/routes/misskey/featured-notes.ts b/lib/routes/misskey/featured-notes.ts index ac7751bba261c8..8712cf3a1e9c77 100644 --- a/lib/routes/misskey/featured-notes.ts +++ b/lib/routes/misskey/featured-notes.ts @@ -1,8 +1,6 @@ import { Route, ViewType } from '@/types'; import got from '@/utils/got'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/notes/featured/:site', @@ -25,9 +23,6 @@ export const route: Route = { async function handler(ctx) { const site = ctx.req.param('site'); - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } // docs on: https://misskey-hub.net/docs/api/endpoints/notes/featured.html const url = `https://${site}/api/notes/featured`; diff --git a/lib/routes/misskey/user-timeline.ts b/lib/routes/misskey/user-timeline.ts index 29fb2f35fc7825..c4ecb066156c53 100644 --- a/lib/routes/misskey/user-timeline.ts +++ b/lib/routes/misskey/user-timeline.ts @@ -1,7 +1,5 @@ import { Route, ViewType } from '@/types'; import utils from './utils'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; import InvalidParameterError from '@/errors/types/invalid-parameter'; export const route: Route = { @@ -29,9 +27,6 @@ async function handler(ctx) { if (!pureUsername || !site) { throw new InvalidParameterError('Provide a valid Misskey username'); } - if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const { accountData } = await utils.getUserTimelineByUsername(pureUsername, site); diff --git a/lib/routes/rsshub/transform/html.ts b/lib/routes/rsshub/transform/html.ts index d63c73782c24b1..bfaaee84ad461c 100644 --- a/lib/routes/rsshub/transform/html.ts +++ b/lib/routes/rsshub/transform/html.ts @@ -1,8 +1,6 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/transform/html/:url/:routeParams', @@ -10,12 +8,6 @@ export const route: Route = { example: '/rsshub/transform/html/https%3A%2F%2Fwechat2rss.xlab.app%2Fposts%2Flist%2F/item=div%5Bclass%3D%27post%2Dcontent%27%5D%20p%20a', parameters: { url: '`encodeURIComponent`ed URL address', routeParams: 'Transformation rules, requires URL encode' }, features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: '', - }, - ], requirePuppeteer: false, antiCrawler: false, supportBT: false, @@ -57,9 +49,6 @@ Specify options (in the format of query string) in parameter \`routeParams\` par }; async function handler(ctx) { - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = ctx.req.param('url'); const response = await got({ method: 'get', diff --git a/lib/routes/rsshub/transform/json.ts b/lib/routes/rsshub/transform/json.ts index ec4ba9601d8982..994c441161b68b 100644 --- a/lib/routes/rsshub/transform/json.ts +++ b/lib/routes/rsshub/transform/json.ts @@ -1,8 +1,6 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; function jsonGet(obj, attr) { if (typeof attr !== 'string') { @@ -22,12 +20,6 @@ export const route: Route = { example: '/rsshub/transform/json/https%3A%2F%2Fapi.github.com%2Frepos%2Fginuerzh%2Fgost%2Freleases/title=Gost%20releases&itemTitle=tag_name&itemLink=html_url&itemDesc=body', parameters: { url: '`encodeURIComponent`ed URL address', routeParams: 'Transformation rules, requires URL encode' }, features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: '', - }, - ], requirePuppeteer: false, antiCrawler: false, supportBT: false, @@ -70,9 +62,6 @@ JSON Path only supports format like \`a.b.c\`. if you need to access arrays, lik }; async function handler(ctx) { - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = ctx.req.param('url'); const response = await got({ method: 'get', diff --git a/lib/routes/rsshub/transform/sitemap.ts b/lib/routes/rsshub/transform/sitemap.ts index f013e943af0241..6994e03ef5eed0 100644 --- a/lib/routes/rsshub/transform/sitemap.ts +++ b/lib/routes/rsshub/transform/sitemap.ts @@ -1,8 +1,6 @@ import { Route } from '@/types'; import got from '@/utils/got'; import { load } from 'cheerio'; -import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; export const route: Route = { path: '/transform/sitemap/:url/:routeParams?', @@ -12,9 +10,6 @@ export const route: Route = { }; async function handler(ctx) { - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } const url = ctx.req.param('url'); const response = await got({ method: 'get', diff --git a/lib/routes/wordpress/index.ts b/lib/routes/wordpress/index.ts index 05f0e4e0f41264..f085fa05eb238d 100644 --- a/lib/routes/wordpress/index.ts +++ b/lib/routes/wordpress/index.ts @@ -5,7 +5,6 @@ import { load } from 'cheerio'; import { parseDate } from '@/utils/parse-date'; import parser from '@/utils/rss-parser'; import { config } from '@/config'; -import ConfigNotFoundError from '@/errors/types/config-not-found'; import { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr } from './util'; @@ -13,10 +12,6 @@ async function handler(ctx) { const { url = 'https://wordpress.org/news', filter } = ctx.req.param(); const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50; - if (!config.feature.allow_user_supply_unsafe_domain) { - throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`); - } - if (!/^(https?):\/\/[^\s#$./?].\S*$/i.test(url)) { throw new Error('Invalid URL'); } @@ -148,19 +143,12 @@ export const route: Route = { You can also subscribe to multiple categories. \`/category/Podcast,Community\` to subscribe to both the Podcast and Community categories. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community). Categories and tags can be combined as well. \`/category/Releases/tag/tagging\` to subscribe to the Releases category and the tagging tag. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging). - + You can also search for keywords. \`/search/Blog\` to search for the keyword "Blog". In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog). :::`, categories: ['blog'], features: { - requireConfig: [ - { - name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN', - description: `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`, - optional: false, - }, - ], requirePuppeteer: false, antiCrawler: false, supportRadar: false, diff --git a/lib/utils/ofetch.test.ts b/lib/utils/ofetch.test.ts index 8815ad2adc641d..2dd29dedeeaf4c 100644 --- a/lib/utils/ofetch.test.ts +++ b/lib/utils/ofetch.test.ts @@ -48,4 +48,20 @@ describe('ofetch', () => { }); expect(response.test).toBe('rsshub'); }); + + it('ban-ip', async () => { + try { + await ofetch('http://192.168.1.1'); + } catch (error: any) { + expect(error.message).toContain('The IP of the domain is reserved!'); + } + }); + + it('ban-ip-domain', async () => { + try { + await ofetch('http://10.0.0.1.nip.io'); + } catch (error: any) { + expect(error.message).toContain('The IP of the domain is reserved!'); + } + }); }); diff --git a/lib/utils/ofetch.ts b/lib/utils/ofetch.ts index 0ea437355eee41..3b0e8ec5a3f19d 100644 --- a/lib/utils/ofetch.ts +++ b/lib/utils/ofetch.ts @@ -1,12 +1,17 @@ import { createFetch } from 'ofetch'; import { config } from '@/config'; import logger from '@/utils/logger'; +import ip from 'ipaddr.js'; +import { lookup as nativeCallbackLookup } from 'dns'; +import { promisify } from 'util'; + +const lookup = promisify(nativeCallbackLookup); const rofetch = createFetch().create({ retryStatusCodes: [400, 408, 409, 425, 429, 500, 502, 503, 504], retry: config.requestRetry, retryDelay: 1000, - // timeout: config.requestTimeout, + timeout: config.requestTimeout, onResponseError({ request, response, options }) { if (options.retry) { logger.warn(`Request ${request} with error ${response.status} remaining retry attempts: ${options.retry}`); @@ -26,6 +31,32 @@ const rofetch = createFetch().create({ headers: { 'user-agent': config.ua, }, + onRequest: async ({ request }) => { + const url = typeof request === 'string' ? new URL(request) : new URL(request.url); + let addr; + // for test + if (url.hostname.endsWith('.test') || url.hostname.endsWith('.mock') || url.hostname.endsWith('.proxy')) { + return; + } + if (ip.isValid(url.hostname)) { + addr = url.hostname; + } else { + const { address } = await lookup(url.hostname); + addr = address; + } + const parsedIp = ip.process(addr); + if (parsedIp.range() === 'unicast') { + return; + } + if (config.feature.allow_cidr) { + const allowCIDR = ip.parseCIDR(config.feature.allow_cidr); + // eslint-disable-next-line unicorn/prefer-regexp-test + if (parsedIp.match(allowCIDR)) { + return; + } + } + throw new Error('The IP of the domain is reserved!'); + }, }); export default rofetch; diff --git a/package.json b/package.json index 3432c39bff7f47..f282b6f1b14f96 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "instagram-private-api": "1.46.1", "ioredis": "5.4.1", "ip-regex": "5.0.0", + "ipaddr.js": "^2.2.0", "jsdom": "25.0.1", "json-bigint": "1.0.0", "jsrsasign": "10.9.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 1d9394c6279beb..e3c591b2ae42f0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -134,6 +134,9 @@ importers: ip-regex: specifier: 5.0.0 version: 5.0.0 + ipaddr.js: + specifier: ^2.2.0 + version: 2.2.0 jsdom: specifier: 25.0.1 version: 25.0.1(bufferutil@4.0.8)(utf-8-validate@5.0.10) @@ -3572,6 +3575,10 @@ packages: resolution: {integrity: sha512-fOCG6lhoKKakwv+C6KdsOnGvgXnmgfmp0myi3bcNwj3qfwPAxRKWEuFhvEFF7ceYIz6+1jRZ+yguLFAmUNPEfw==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + ipaddr.js@2.2.0: + resolution: {integrity: sha512-Ag3wB2o37wslZS19hZqorUnrnzSkpOVy+IiiDEiTqNubEYpYuHWIf6K4psgN2ZWKExS4xhVCrRVfb/wfW8fWJA==} + engines: {node: '>= 10'} + is-arrayish@0.2.1: resolution: {integrity: sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==} @@ -9365,6 +9372,8 @@ snapshots: ip-regex@5.0.0: {} + ipaddr.js@2.2.0: {} + is-arrayish@0.2.1: {} is-arrayish@0.3.2: {}