Skip to content

Commit

Permalink
feat(middleware): 优化 多媒体处理增加 data.image/item.enclosure_url/item.image… (
Browse files Browse the repository at this point in the history 
#16610)

* feat(middleware): 优化 多媒体处理增加 data.image/item.enclosure_url/item.image/item.itunes_item_image 等字段

* test(middleware): 补充 anti-hotlink 缺失的测试用例
  • Loading branch information
@CaoMeiYouRen
CaoMeiYouRen authored Sep 5, 2024
1 parent dc9d3b7 commit ed8b868
Show file tree
Hide file tree
Showing 3 changed files with 263 additions and 13 deletions.
197 changes: 191 additions & 6 deletions lib/middleware/anti-hotlink.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,148 @@ const expects = {
desc: '<video src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub"></video> - Powered by RSSHub',
},
},
extraComplicated: {
origin: {
items: [
{
content:
'<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n\n<a href="http://mock.com/DIYgod/RSSHub"></a>\n<img src="https://mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">\n<img data-src="/DIYgod/RSSHub.jpg" src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<img data-mock="/DIYgod/RSSHub.png" src="https://mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">\n<img mock="/DIYgod/RSSHub.gif" src="https://mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">\n<img src="http://mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">\n<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" referrerpolicy="no-referrer">',
itunes: {},
},
{
content: '<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">',
itunes: {},
},
{
content:
'<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" referrerpolicy="no-referrer">',
enclosure: {
url: 'https://mock.com/DIYgod/RSSHub.png',
type: 'image/png',
},
itunes: {
image: 'https://mock.com/DIYgod/RSSHub.gif',
},
},
],
image: {
link: 'https://github.com/DIYgod/RSSHub',
url: 'https://mock.com/DIYgod/RSSHub.png',
title: 'Test complicated',
},
description: '<img src="http://mock.com/DIYgod/DIYgod/RSSHub"> - Powered by RSSHub',
},
processed: {
items: [
{
content:
'<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n\n<a href="http://mock.com/DIYgod/RSSHub"></a>\n<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">\n<img data-src="/DIYgod/RSSHub.jpg" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<img data-mock="/DIYgod/RSSHub.png" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.png" referrerpolicy="no-referrer">\n<img mock="/DIYgod/RSSHub.gif" src="https://i3.wp.com/mock.com/DIYgod/RSSHub.gif" referrerpolicy="no-referrer">\n<img src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub" referrerpolicy="no-referrer">\n<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" referrerpolicy="no-referrer">',
itunes: {},
},
{
content: '<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">',
itunes: {},
},
{
content:
'<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" referrerpolicy="no-referrer">',
enclosure: {
url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.png',
type: 'image/png',
},
itunes: {
image: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.gif',
},
},
],
image: {
link: 'https://github.com/DIYgod/RSSHub',
url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.png',
title: 'Test complicated',
},
description: '<img src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub"> - Powered by RSSHub',
},
urlencoded: {
items: [
{
content:
'<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.jpg" referrerpolicy="no-referrer">\n\n<a href="http://mock.com/DIYgod/RSSHub"></a>\n<img src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.jpg" data-src="/DIYgod/RSSHub0.jpg" referrerpolicy="no-referrer">\n<img data-src="/DIYgod/RSSHub.jpg" src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.jpg" referrerpolicy="no-referrer">\n<img data-mock="/DIYgod/RSSHub.png" src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.png" referrerpolicy="no-referrer">\n<img mock="/DIYgod/RSSHub.gif" src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.gif" referrerpolicy="no-referrer">\n<img src="https://images.weserv.nl?url=http%3A%2F%2Fmock.com%2FDIYgod%2FDIYgod%2FRSSHub" referrerpolicy="no-referrer">\n<img src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.jpg" referrerpolicy="no-referrer">\n<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" referrerpolicy="no-referrer">',
itunes: {},
},
{
content: '<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.jpg" referrerpolicy="no-referrer">',
itunes: {},
},
{
content:
'<a href="https://mock.com/DIYgod/RSSHub"></a>\n<img src="https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.jpg" referrerpolicy="no-referrer">\n<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==" referrerpolicy="no-referrer">',
enclosure: {
url: 'https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.png',
type: 'image/png',
},
itunes: {
image: 'https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.gif',
},
},
],
image: {
link: 'https://github.com/DIYgod/RSSHub',
url: 'https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.png',
title: 'Test complicated',
},
description: '<img src="https://images.weserv.nl?url=http%3A%2F%2Fmock.com%2FDIYgod%2FDIYgod%2FRSSHub"> - Powered by RSSHub',
},
},
extraMultimedia: {
origin: {
items: [
{
content:
'<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<video src="https://mock.com/DIYgod/RSSHub.mp4"></video>\n<video poster="https://mock.com/DIYgod/RSSHub.jpg">\n<source src="https://mock.com/DIYgod/RSSHub.mp4" type="video/mp4">\n<source src="https://mock.com/DIYgod/RSSHub.webm" type="video/webm">\n</video>\n<audio src="https://mock.com/DIYgod/RSSHub.mp3"></audio>\n<iframe src="https://mock.com/DIYgod/RSSHub.html" referrerpolicy="no-referrer"></iframe>',
},
{
content: '<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<video src="https://mock.com/DIYgod/RSSHub.mp4"></video>',
enclosure: {
url: 'https://mock.com/DIYgod/RSSHub.mp4',
type: 'video/mp4',
},
},
],
description: '<video src="http://mock.com/DIYgod/DIYgod/RSSHub"></video> - Powered by RSSHub',
},
relayed: {
items: [
{
content:
'<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<video src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4"></video>\n<video poster="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg">\n<source src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4" type="video/mp4">\n<source src="https://i3.wp.com/mock.com/DIYgod/RSSHub.webm" type="video/webm">\n</video>\n<audio src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp3"></audio>\n<iframe src="https://mock.com/DIYgod/RSSHub.html" referrerpolicy="no-referrer"></iframe>',
},
{
content: '<img src="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<video src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4"></video>',
enclosure: {
url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4',
type: 'video/mp4',
},
},
],
description: '<video src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub"></video> - Powered by RSSHub',
},
partlyRelayed: {
items: [
{
content:
'<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<video src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4"></video>\n<video poster="https://i3.wp.com/mock.com/DIYgod/RSSHub.jpg">\n<source src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4" type="video/mp4">\n<source src="https://i3.wp.com/mock.com/DIYgod/RSSHub.webm" type="video/webm">\n</video>\n<audio src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp3"></audio>\n<iframe src="https://mock.com/DIYgod/RSSHub.html" referrerpolicy="no-referrer"></iframe>',
},
{
content: '<img src="https://mock.com/DIYgod/RSSHub.jpg" referrerpolicy="no-referrer">\n<video src="https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4"></video>',
enclosure: {
url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4',
type: 'video/mp4',
},
},
],
description: '<video src="https://i3.wp.com/mock.com/DIYgod/DIYgod/RSSHub"></video> - Powered by RSSHub',
},
},
};

const testAntiHotlink = async (path, expectObj, query?: string | Record<string, any>) => {
Expand All @@ -142,12 +284,55 @@ const testAntiHotlink = async (path, expectObj, query?: string | Record<string,
return parsed;
};

const expectImgOrigin = (query?: string | Record<string, any>) => testAntiHotlink('/test/complicated', expects.complicated.origin, query);
const expectImgProcessed = (query?: string | Record<string, any>) => testAntiHotlink('/test/complicated', expects.complicated.processed, query);
const expectImgUrlencoded = (query?: string | Record<string, any>) => testAntiHotlink('/test/complicated', expects.complicated.urlencoded, query);
const expectMultimediaOrigin = (query?: string | Record<string, any>) => testAntiHotlink('/test/multimedia', expects.multimedia.origin, query);
const expectMultimediaRelayed = (query?: string | Record<string, any>) => testAntiHotlink('/test/multimedia', expects.multimedia.relayed, query);
const expectMultimediaPartlyRelayed = (query?: string | Record<string, any>) => testAntiHotlink('/test/multimedia', expects.multimedia.partlyRelayed, query);
const testAntiHotlinkExtra = async (path, expectObj, query?: string | Record<string, any>) => {
const app = (await import('@/app')).default;

path += query ? `?${new URLSearchParams(query).toString()}` : '';

const response = await app.request(path);
const parsed = await parser.parseString(await response.text());
const obj = {
description: parsed.description,
image: parsed.image,
items: parsed.items.slice(0, expectObj.items.length).map((e) => ({
content: e.content,
enclosure: e.enclosure,
itunes: e.itunes,
})),
};
expect(obj).toEqual(expectObj);

return parsed;
};

const expectImgOrigin = async (query?: string | Record<string, any>) => {
await testAntiHotlink('/test/complicated', expects.complicated.origin, query);
await testAntiHotlinkExtra('/test/complicated', expects.extraComplicated.origin, query);
};
const expectImgProcessed = async (query?: string | Record<string, any>) => {
await testAntiHotlink('/test/complicated', expects.complicated.processed, query);
await testAntiHotlinkExtra('/test/complicated', expects.extraComplicated.processed, query);
};

const expectImgUrlencoded = async (query?: string | Record<string, any>) => {
await testAntiHotlink('/test/complicated', expects.complicated.urlencoded, query);
await testAntiHotlinkExtra('/test/complicated', expects.extraComplicated.urlencoded, query);
};

const expectMultimediaOrigin = async (query?: string | Record<string, any>) => {
await testAntiHotlink('/test/multimedia', expects.multimedia.origin, query);
await testAntiHotlinkExtra('/test/multimedia', expects.extraMultimedia.origin, query);
};

const expectMultimediaRelayed = async (query?: string | Record<string, any>) => {
await testAntiHotlink('/test/multimedia', expects.multimedia.relayed, query);
await testAntiHotlinkExtra('/test/multimedia', expects.extraMultimedia.relayed, query);
};

const expectMultimediaPartlyRelayed = async (query?: string | Record<string, any>) => {
await testAntiHotlink('/test/multimedia', expects.multimedia.partlyRelayed, query);
await testAntiHotlinkExtra('/test/multimedia', expects.extraMultimedia.partlyRelayed, query);
};

describe('anti-hotlink', () => {
it('template-legacy', async () => {
Expand Down
28 changes: 28 additions & 0 deletions lib/middleware/anti-hotlink.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,18 @@ const parseUrl = (str: string) => {

return url;
};

const replaceUrl = (template?: string, url?: string) => {
if (!template || !url) {
return url;
}
const oldUrl = parseUrl(url);
if (oldUrl && oldUrl.protocol !== 'data:') {
return interpolate(template, oldUrl);
}
return url;
};

const replaceUrls = ($: CheerioAPI, selector: string, template: string, attribute = 'src') => {
$(selector).each(function () {
const oldSrc = $(this).attr(attribute);
Expand Down Expand Up @@ -120,6 +132,9 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
// image link
const data: Data = ctx.get('data');
if (data) {
if (data.image) {
data.image = replaceUrl(imageHotlinkTemplate, data.image);
}
if (data.description) {
data.description = process(data.description, imageHotlinkTemplate, multimediaHotlinkTemplate);
}
Expand All @@ -129,6 +144,19 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
if (item.description) {
item.description = process(item.description, imageHotlinkTemplate, multimediaHotlinkTemplate);
}
if (item.enclosure_url && item.enclosure_type) {
if (item.enclosure_type.startsWith('image/')) {
item.enclosure_url = replaceUrl(imageHotlinkTemplate, item.enclosure_url);
} else if (/^(video|audio)\//.test(item.enclosure_type)) {
item.enclosure_url = replaceUrl(multimediaHotlinkTemplate, item.enclosure_url);
}
}
if (item.image) {
item.image = replaceUrl(imageHotlinkTemplate, item.image);
}
if (item.itunes_item_image) {
item.itunes_item_image = replaceUrl(imageHotlinkTemplate, item.itunes_item_image);
}
}
}

Expand Down
51 changes: 44 additions & 7 deletions lib/routes/test/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ async function handler(ctx) {
throw new InvalidParameterError('Test invalid parameter error');
}
let item: DataItem[] = [];
let image: string | null = null;
switch (ctx.req.param('id')) {
case 'filter':
item = [
Expand Down Expand Up @@ -141,6 +142,7 @@ async function handler(ctx) {
break;
}
case 'complicated':
image = 'https://mock.com/DIYgod/RSSHub.png';
item.push(
{
title: `Complicated Title`,
Expand All @@ -166,26 +168,60 @@ async function handler(ctx) {
pubDate: new Date(`2019-3-1`).toUTCString(),
link: `https://mock.com/DIYgod/RSSHub`,
author: `DIYgod`,
},
{
title: `Complicated Title`,
description: `<a href="/DIYgod/RSSHub"></a>
<img src="/DIYgod/RSSHub.jpg">
<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==">`,
pubDate: new Date(`2019-3-1`).toUTCString(),
link: `//mock.com/DIYgod/RSSHub`,
author: `DIYgod`,
enclosure_url: 'https://mock.com/DIYgod/RSSHub.png',
enclosure_type: 'image/png',
itunes_item_image: 'https://mock.com/DIYgod/RSSHub.gif',
},
{
title: `Complicated Title`,
description: `<a href="/DIYgod/RSSHub"></a>
<img src="/DIYgod/RSSHub.jpg">
<img src="data:image/gif;base64,R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==">`,
pubDate: new Date(`2019-3-1`).toUTCString(),
link: `//mock.com/DIYgod/RSSHub`,
author: `DIYgod`,
image: 'https://mock.com/DIYgod/RSSHub.jpg',
}
);

break;

case 'multimedia':
item.push({
title: `Multimedia Title`,
description: `<img src="/DIYgod/RSSHub.jpg">
item.push(
{
title: `Multimedia Title`,
description: `<img src="/DIYgod/RSSHub.jpg">
<video src="/DIYgod/RSSHub.mp4"></video>
<video poster="/DIYgod/RSSHub.jpg">
<source src="/DIYgod/RSSHub.mp4" type="video/mp4">
<source src="/DIYgod/RSSHub.webm" type="video/webm">
</video>
<audio src="/DIYgod/RSSHub.mp3"></audio>
<iframe src="/DIYgod/RSSHub.html"></iframe>`,
pubDate: new Date(`2019-3-1`).toUTCString(),
link: `https://mock.com/DIYgod/RSSHub`,
author: `DIYgod`,
});
pubDate: new Date(`2019-3-1`).toUTCString(),
link: `https://mock.com/DIYgod/RSSHub`,
author: `DIYgod`,
},
{
title: `Multimedia Title`,
description: `<img src="/DIYgod/RSSHub.jpg">
<video src="/DIYgod/RSSHub.mp4"></video>`,
pubDate: new Date(`2019-3-1`).toUTCString(),
link: `https://mock.com/DIYgod/RSSHub`,
author: `DIYgod`,
enclosure_url: 'https://mock.com/DIYgod/RSSHub.mp4',
enclosure_type: 'video/mp4',
}
);

break;

Expand Down Expand Up @@ -369,6 +405,7 @@ async function handler(ctx) {
}

return {
image,
title: `Test ${ctx.req.param('id')}`,
itunes_author: ctx.req.param('id') === 'enclosure' ? 'DIYgod' : null,
link: 'https://github.com/DIYgod/RSSHub',
Expand Down

0 comments on commit ed8b868

Please sign in to comment.