diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts index 54244239..9b35597d 100644 --- a/src/parser/content-evaluator-module.ts +++ b/src/parser/content-evaluator-module.ts @@ -10,8 +10,8 @@ import { IssueActivity } from "../issue-activity"; import { GithubCommentScore, Module, Result } from "./processor"; import { Value } from "@sinclair/typebox/value"; import { commentEnum, CommentKind, CommentType } from "../configuration/comment-types"; -import { Type } from "@sinclair/typebox"; import logger from "../helpers/logger"; +import openAiRelevanceResponseSchema, { RelevancesByOpenAi } from "../types/openai-type"; type CommentToEvaluate = { id: number; comment: string }; type ReviewCommentToEvaluate = { id: number; comment: string; diff_hunk: string }; @@ -76,31 +76,7 @@ export class ContentEvaluatorModule implements Module { async _processComment(comments: Readonly[], specificationBody: string) { const commentsWithScore: GithubCommentScore[] = [...comments]; - - // exclude comments that have fixed relevance multiplier. e.g. review comments = 1 - const commentsToEvaluate: CommentToEvaluate[] = []; - const reviewCommentsToEvaluate: ReviewCommentToEvaluate[] = []; - for (let i = 0; i < commentsWithScore.length; i++) { - const currentComment = commentsWithScore[i]; - if (!this._fixedRelevances[currentComment.type]) { - if (currentComment.type & CommentKind.PULL) { - if (currentComment?.diff_hunk) { - //Eval PR comment with diff_hunk, all other PR comments get relevance:1 by default - - reviewCommentsToEvaluate.push({ - id: currentComment.id, - comment: currentComment.content, - diff_hunk: currentComment.diff_hunk, - }); - } - } else { - commentsToEvaluate.push({ - id: currentComment.id, - comment: currentComment.content, - }); - } - } - } + const { commentsToEvaluate, reviewCommentsToEvaluate } = this._splitCommentsByPrompt(commentsWithScore); const relevancesByAI = await this._evaluateComments( specificationBody, @@ -114,7 +90,7 @@ export class ContentEvaluatorModule implements Module { for (let i = 0; i < commentsWithScore.length; i++) { const currentComment = commentsWithScore[i]; - let currentRelevance = 1; // For comments not in fixed relevance types or missed by OpenAI evaluation + let currentRelevance = 1; // For comments not in fixed relevance types and missed by OpenAI evaluation if (this._fixedRelevances[currentComment.type]) { currentRelevance = this._fixedRelevances[currentComment.type]; @@ -133,11 +109,39 @@ export class ContentEvaluatorModule implements Module { return commentsWithScore; } + _splitCommentsByPrompt(commentsWithScore: Readonly[]) { + // exclude comments that have fixed relevance multiplier. e.g. review comments = 1 + const commentsToEvaluate: CommentToEvaluate[] = []; + const reviewCommentsToEvaluate: ReviewCommentToEvaluate[] = []; + for (let i = 0; i < commentsWithScore.length; i++) { + const currentComment = commentsWithScore[i]; + if (!this._fixedRelevances[currentComment.type]) { + if (currentComment.type & CommentKind.PULL) { + if (currentComment?.diff_hunk) { + //Eval PR comment with diff_hunk, all other PR comments get relevance:1 by default + + reviewCommentsToEvaluate.push({ + id: currentComment.id, + comment: currentComment.content, + diff_hunk: currentComment.diff_hunk, + }); + } + } else { + commentsToEvaluate.push({ + id: currentComment.id, + comment: currentComment.content, + }); + } + } + } + return { commentsToEvaluate, reviewCommentsToEvaluate }; + } + async _evaluateComments( specification: string, comments: CommentToEvaluate[], reviewComments: ReviewCommentToEvaluate[] - ): Promise { + ): Promise { let combinedRelevances: Relevances = {}; if (comments.length) { @@ -176,14 +180,13 @@ export class ContentEvaluatorModule implements Module { const jsonResponse = JSON.parse(rawResponse); - const responseType = Type.Record(Type.String(), Type.Number({ minimum: 0, maximum: 1 })); - - if (Value.Check(responseType, jsonResponse)) { - const relevances = Value.Convert(responseType, jsonResponse) as { [k: string]: number }; + try { + const relevances = Value.Decode(openAiRelevanceResponseSchema, jsonResponse); logger.info(`Relevances by OpenAI: ${JSON.stringify(relevances)}`); return relevances; - } else { - throw new Error(`Invalid response type received from openai while evaluating: ${jsonResponse}`); + } catch (e) { + logger.error(`Invalid response type received from openai while evaluating: ${jsonResponse} \n\nError: ${e}`); + throw new Error("Error in evaluation by OpenAI."); } } diff --git a/src/types/openai-type.ts b/src/types/openai-type.ts new file mode 100644 index 00000000..58190740 --- /dev/null +++ b/src/types/openai-type.ts @@ -0,0 +1,7 @@ +import { Type, Static } from "@sinclair/typebox"; + +const openAiRelevanceResponseSchema = Type.Record(Type.String(), Type.Number({ minimum: 0, maximum: 1 })); + +export type RelevancesByOpenAi = Static; + +export default openAiRelevanceResponseSchema;