From 5db3e3a661ecbdb6cfdcf55ea9957df02aa522c0 Mon Sep 17 00:00:00 2001 From: Fleurxxx <984209872@qq.com> Date: Sat, 17 Aug 2024 19:32:54 +0800 Subject: [PATCH 1/9] feat: Invoke the image upload and image recognition interface to realize the function of carrying picture dialogue --- app/controller/app-center/aiChat.ts | 8 ++++++ app/router/appCenter/base.ts | 22 +++++++-------- app/service/app-center/aiChat.ts | 42 +++++++++++++++++++++++++++++ config/config.default.ts | 34 ++++++++++++++++++++++- 4 files changed, 94 insertions(+), 12 deletions(-) diff --git a/app/controller/app-center/aiChat.ts b/app/controller/app-center/aiChat.ts index 9527c80..350af76 100644 --- a/app/controller/app-center/aiChat.ts +++ b/app/controller/app-center/aiChat.ts @@ -23,6 +23,14 @@ export default class AiChatController extends Controller { const model = foundationModel?.model ?? E_FOUNDATION_MODEL.GPT_35_TURBO; const token = foundationModel.token; ctx.body = await ctx.service.appCenter.aiChat.getAnswerFromAi(messages, { model, token }); + } + + public async uploadFile() { + const { ctx } = this; + const fileStream = await ctx.getFileStream(); + const foundationModelObject = JSON.parse(fileStream.fields.foundationModel); + const { model, token } = foundationModelObject.foundationModel; + ctx.body = await ctx.service.appCenter.aiChat.getFileContentFromAi(fileStream, { model, token }); } } diff --git a/app/router/appCenter/base.ts b/app/router/appCenter/base.ts index c406442..2b7c273 100644 --- a/app/router/appCenter/base.ts +++ b/app/router/appCenter/base.ts @@ -1,14 +1,14 @@ /** -* Copyright (c) 2023 - present TinyEngine Authors. -* Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. -* -* Use of this source code is governed by an MIT-style license. -* -* THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, -* BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR -* A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. -* -*/ + * Copyright (c) 2023 - present TinyEngine Authors. + * Copyright (c) 2023 - present Huawei Cloud Computing Technologies Co., Ltd. + * + * Use of this source code is governed by an MIT-style license. + * + * THE OPEN SOURCE SOFTWARE IN THIS PRODUCT IS DISTRIBUTED IN THE HOPE THAT IT WILL BE USEFUL, + * BUT WITHOUT ANY WARRANTY, WITHOUT EVEN THE IMPLIED WARRANTY OF MERCHANTABILITY OR FITNESS FOR + * A PARTICULAR PURPOSE. SEE THE APPLICABLE LICENSES FOR MORE DETAILS. + * + */ import { Application } from 'egg'; export default (app: Application) => { @@ -22,7 +22,6 @@ export default (app: Application) => { const subRouter = router.namespace(ROUTER_PREFIX); // 应用管理 - subRouter.get('/apps/detail/:id', controller.appCenter.apps.detail); subRouter.post('/apps/update/:id', controller.appCenter.apps.update); @@ -114,4 +113,5 @@ export default (app: Application) => { // AI大模型聊天接口 subRouter.post('/ai/chat', controller.appCenter.aiChat.aiChat); + subRouter.post('/ai/files', controller.appCenter.aiChat.uploadFile); }; diff --git a/app/service/app-center/aiChat.ts b/app/service/app-center/aiChat.ts index 85d3edf..61f7a8c 100644 --- a/app/service/app-center/aiChat.ts +++ b/app/service/app-center/aiChat.ts @@ -162,5 +162,47 @@ export default class AiChat extends Service { } return messages; } + + /** + * 文件上传 + * + * @param model + * @return + */ + + async getFileContentFromAi(fileStream: any, chatConfig: any) { + const answer = await this.requestFileContentFromAi(fileStream, chatConfig); + return this.ctx.helper.getResponseData({ + originalResponse: answer + }); + } + + async requestFileContentFromAi(file: any, chatConfig: any) { + const { ctx } = this; + let res: any = null; + try { + // 文件上传 + const aiUploadConfig = this.config.uploadFile(file, chatConfig.token); + const { httpRequestUrl, httpRequestOption } = aiUploadConfig[chatConfig.model]; + this.ctx.logger.debug(httpRequestOption); + res = await ctx.curl(httpRequestUrl, httpRequestOption); + const imageObject = JSON.parse(res.res.data.toString()); + const fileObject = imageObject.data[0].id; + // 文件解析 + const imageAnalysisConfig = this.config.parsingFile(fileObject, chatConfig.token); + const { analysisImageHttpRequestUrl, analysisImageHttpRequestOption } = imageAnalysisConfig[chatConfig.model]; + res = await ctx.curl(analysisImageHttpRequestUrl, analysisImageHttpRequestOption); + res.data = JSON.parse(res.res.data.toString()); + } catch (e: any) { + this.ctx.logger.debug(`调用上传图片接口失败: ${(e as Error).message}`); + return this.ctx.helper.getResponseData(`调用上传图片接口失败: ${(e as Error).message}`); + } + + if (!res) { + return this.ctx.helper.getResponseData(`调用上传图片接口未返回正确数据.`); + } + + return res.data; + } } diff --git a/config/config.default.ts b/config/config.default.ts index ac0d902..d078264 100644 --- a/config/config.default.ts +++ b/config/config.default.ts @@ -14,7 +14,6 @@ import { EggAppConfig, PowerPartial } from 'egg'; import { I_SchemaConvert } from '../app/lib/interface'; import { E_SchemaFormatFunc, E_FOUNDATION_MODEL } from '../app/lib/enum'; - export default (appInfo) => { const config = {} as PowerPartial; @@ -305,6 +304,39 @@ export default (appInfo) => { }; }; + // 文件上传接口 + config.uploadFile = (file: any, token: string) => { + return { + [E_FOUNDATION_MODEL.MOONSHOT_V1_8K]: { + httpRequestUrl: `https://api.moonshot.cn/v1/files`, + httpRequestOption: { + data: { + file: file, + purpose: 'file-extract' + }, + headers: { + Authorization: `Bearer ${token}` + } + } + } + }; + }; + + // 文件解析接口 + config.parsingFile = (fileId: any, token: string) => { + return { + [E_FOUNDATION_MODEL.MOONSHOT_V1_8K]: { + analysisImageHttpRequestUrl: `https://api.moonshot.cn/v1/files/${fileId}/content`, + analysisImageHttpRequestOption: { + method: 'GET', + headers: { + Authorization: `Bearer ${token}` + } + } + } + }; + }; + config.npmRegistryOptions = [ '--registry=https://registry.npmjs.org/' ]; From 85989cd871dcbf2884dd18ad4300f505cc94465b Mon Sep 17 00:00:00 2001 From: wangjunyue <984209872@qq.com> Date: Sun, 29 Sep 2024 18:33:04 +0800 Subject: [PATCH 2/9] feat: Create optimized branch --- app/service/app-center/aiChat.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/service/app-center/aiChat.ts b/app/service/app-center/aiChat.ts index 61f7a8c..915aff7 100644 --- a/app/service/app-center/aiChat.ts +++ b/app/service/app-center/aiChat.ts @@ -53,6 +53,7 @@ export default class AiChat extends Service { const aiChatConfig = this.config.aiChat(messages, chatConfig.token); const { httpRequestUrl, httpRequestOption } = aiChatConfig[chatConfig.model]; this.ctx.logger.debug(httpRequestOption); + res = await ctx.curl(httpRequestUrl, httpRequestOption); } catch (e: any) { this.ctx.logger.debug(`调用AI大模型接口失败: ${(e as Error).message}`); From 6c8fb93f1fb91d82e8a4cf5b711db1f0a534ce92 Mon Sep 17 00:00:00 2001 From: wangjunyue <984209872@qq.com> Date: Mon, 21 Oct 2024 02:09:33 +0800 Subject: [PATCH 3/9] fix: file upload bug --- app/service/app-center/aiChat.ts | 51 +++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 11 deletions(-) diff --git a/app/service/app-center/aiChat.ts b/app/service/app-center/aiChat.ts index 915aff7..06bd69b 100644 --- a/app/service/app-center/aiChat.ts +++ b/app/service/app-center/aiChat.ts @@ -12,6 +12,15 @@ import { Service } from 'egg'; import Transformer from '@opentiny/tiny-engine-transform'; import { E_FOUNDATION_MODEL } from '../../lib/enum'; +import * as fs from 'fs'; +import * as path from 'path'; + +const OpenAI = require('openai'); + +const client = new OpenAI({ + apiKey: 'sk-uORejJaJs7ZXP6MFQkY9FzYhgPWZms1iOTCBUHA0ipQJRhRt', + baseURL: 'https://api.moonshot.cn/v1' +}); export type AiMessage = { role: string; // 角色 @@ -19,6 +28,11 @@ export type AiMessage = { content: string; // 聊天内容 }; +interface ConfigModel { + model: string; + token: string; +} + export default class AiChat extends Service { /** * 获取ai的答复 @@ -53,7 +67,7 @@ export default class AiChat extends Service { const aiChatConfig = this.config.aiChat(messages, chatConfig.token); const { httpRequestUrl, httpRequestOption } = aiChatConfig[chatConfig.model]; this.ctx.logger.debug(httpRequestOption); - + res = await ctx.curl(httpRequestUrl, httpRequestOption); } catch (e: any) { this.ctx.logger.debug(`调用AI大模型接口失败: ${(e as Error).message}`); @@ -171,32 +185,47 @@ export default class AiChat extends Service { * @return */ - async getFileContentFromAi(fileStream: any, chatConfig: any) { + async getFileContentFromAi(fileStream: any, chatConfig: ConfigModel) { const answer = await this.requestFileContentFromAi(fileStream, chatConfig); return this.ctx.helper.getResponseData({ originalResponse: answer }); } - async requestFileContentFromAi(file: any, chatConfig: any) { + async requestFileContentFromAi(file: any, chatConfig: ConfigModel) { const { ctx } = this; + // // @ts-ignore + const filename = Math.random().toString(36).substr(2) + new Date().getTime() + path.extname(file.filename).toLocaleLowerCase(); + const savePath = path.join(__dirname, filename); + const writeStream = fs.createWriteStream(savePath); + file.pipe(writeStream); + await new Promise(resolve => writeStream.on('close', resolve)); + let res: any = null; try { - // 文件上传 - const aiUploadConfig = this.config.uploadFile(file, chatConfig.token); - const { httpRequestUrl, httpRequestOption } = aiUploadConfig[chatConfig.model]; - this.ctx.logger.debug(httpRequestOption); - res = await ctx.curl(httpRequestUrl, httpRequestOption); - const imageObject = JSON.parse(res.res.data.toString()); - const fileObject = imageObject.data[0].id; + //上传文件 + const fileObject = await client.files.create({ + file: fs.createReadStream(savePath), + purpose: 'file-extract' + }); + // 文件解析 - const imageAnalysisConfig = this.config.parsingFile(fileObject, chatConfig.token); + const imageAnalysisConfig = this.config.parsingFile(fileObject.id, chatConfig.token); const { analysisImageHttpRequestUrl, analysisImageHttpRequestOption } = imageAnalysisConfig[chatConfig.model]; res = await ctx.curl(analysisImageHttpRequestUrl, analysisImageHttpRequestOption); res.data = JSON.parse(res.res.data.toString()); + console.log(res.data); } catch (e: any) { this.ctx.logger.debug(`调用上传图片接口失败: ${(e as Error).message}`); return this.ctx.helper.getResponseData(`调用上传图片接口失败: ${(e as Error).message}`); + } finally { + // 删除本地文件 + try { + await fs.promises.unlink(savePath); + console.log('文件已删除:', savePath); + } catch (err) { + console.error('文件删除失败:', err); + } } if (!res) { From 4eea1d46b97d213ff605a24c5b6e83483d37ab1b Mon Sep 17 00:00:00 2001 From: wangjunyue <984209872@qq.com> Date: Sun, 20 Oct 2024 12:35:38 +0800 Subject: [PATCH 4/9] =?UTF-8?q?feat=EF=BC=9AAdd=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/service/app-center/aiChat.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/service/app-center/aiChat.ts b/app/service/app-center/aiChat.ts index 06bd69b..5cf639e 100644 --- a/app/service/app-center/aiChat.ts +++ b/app/service/app-center/aiChat.ts @@ -67,7 +67,6 @@ export default class AiChat extends Service { const aiChatConfig = this.config.aiChat(messages, chatConfig.token); const { httpRequestUrl, httpRequestOption } = aiChatConfig[chatConfig.model]; this.ctx.logger.debug(httpRequestOption); - res = await ctx.curl(httpRequestUrl, httpRequestOption); } catch (e: any) { this.ctx.logger.debug(`调用AI大模型接口失败: ${(e as Error).message}`); From 3521e128b8cf1c6ee7bcc976fdd11429166a83ee Mon Sep 17 00:00:00 2001 From: wangjunyue <984209872@qq.com> Date: Sat, 26 Oct 2024 18:05:37 +0800 Subject: [PATCH 5/9] refactor: Extract the schema part of the reply content --- app/service/app-center/aiChat.ts | 41 +++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/app/service/app-center/aiChat.ts b/app/service/app-center/aiChat.ts index 5cf639e..d866a8e 100644 --- a/app/service/app-center/aiChat.ts +++ b/app/service/app-center/aiChat.ts @@ -10,7 +10,6 @@ * */ import { Service } from 'egg'; -import Transformer from '@opentiny/tiny-engine-transform'; import { E_FOUNDATION_MODEL } from '../../lib/enum'; import * as fs from 'fs'; import * as path from 'path'; @@ -26,6 +25,7 @@ export type AiMessage = { role: string; // 角色 name?: string; // 名称 content: string; // 聊天内容 + partial?: boolean; }; interface ConfigModel { @@ -45,11 +45,26 @@ export default class AiChat extends Service { */ async getAnswerFromAi(messages: Array, chatConfig: any) { - const answer = await this.requestAnswerFromAi(messages, chatConfig); - const answerContent = answer.choices[0]?.message.content; - // 从ai回复中提取页面的代码 - const codes = this.extractCode(answerContent); - const schema = codes ? Transformer.translate(codes) : null; + let res = await this.requestAnswerFromAi(messages, chatConfig); + let answerContent = ''; + // 若内容过长被截断，继续回复 + if (res.choices[0].finish_reason == 'length') { + const prefix = res.choices[0].message.content; + answerContent += prefix; + messages.push({ + role: 'assistant', + content: prefix, + partial: true + }); + res = await this.requestAnswerFromAi(messages, chatConfig); + answerContent += res.choices[0]?.message.content; + } + const code = this.extractCode(answerContent); + const schema = this.extractSchemaCode(code); + const answer = { + role: res.choices[0]?.message.role, + content: answerContent + }; const replyWithoutCode = this.removeCode(answerContent); return this.ctx.helper.getResponseData({ originalResponse: answer, @@ -137,6 +152,20 @@ export default class AiChat extends Service { return content.substring(0, start) + '<代码在画布中展示>' + content.substring(end); } + private extractSchemaCode(content) { + const startMarker = /```json/; + const endMarker = /```/; + + const start = content.search(startMarker); + const end = content.slice(start + 7).search(endMarker) + start + 7; + + if (start >= 0 && end >= 0) { + return JSON.parse(content.substring(start + 7, end).trim()); + } + + return null; + } + private getStartAndEnd(str: string) { const start = str.search(/```|