Merge branch 'development' of https://github.com/ubiquibot/conversati…

…on-rewards into development
EresDevOrg · Aug 13, 2024 · db161e9 · db161e9
2 parents 4af0362 + 369d614
commit db161e9
Show file tree

Hide file tree

Showing 2 changed files with 44 additions and 34 deletions.
diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts
@@ -10,8 +10,8 @@ import { IssueActivity } from "../issue-activity";
 import { GithubCommentScore, Module, Result } from "./processor";
 import { Value } from "@sinclair/typebox/value";
 import { commentEnum, CommentKind, CommentType } from "../configuration/comment-types";
-import { Type } from "@sinclair/typebox";
 import logger from "../helpers/logger";
+import openAiRelevanceResponseSchema, { RelevancesByOpenAi } from "../types/openai-type";
 
 type CommentToEvaluate = { id: number; comment: string };
 type ReviewCommentToEvaluate = { id: number; comment: string; diff_hunk: string };
@@ -76,31 +76,7 @@ export class ContentEvaluatorModule implements Module {
 
   async _processComment(comments: Readonly<GithubCommentScore>[], specificationBody: string) {
     const commentsWithScore: GithubCommentScore[] = [...comments];
-
-    // exclude comments that have fixed relevance multiplier. e.g. review comments = 1
-    const commentsToEvaluate: CommentToEvaluate[] = [];
-    const reviewCommentsToEvaluate: ReviewCommentToEvaluate[] = [];
-    for (let i = 0; i < commentsWithScore.length; i++) {
-      const currentComment = commentsWithScore[i];
-      if (!this._fixedRelevances[currentComment.type]) {
-        if (currentComment.type & CommentKind.PULL) {
-          if (currentComment?.diff_hunk) {
-            //Eval PR comment with diff_hunk, all other PR comments get relevance:1 by default
-
-            reviewCommentsToEvaluate.push({
-              id: currentComment.id,
-              comment: currentComment.content,
-              diff_hunk: currentComment.diff_hunk,
-            });
-          }
-        } else {
-          commentsToEvaluate.push({
-            id: currentComment.id,
-            comment: currentComment.content,
-          });
-        }
-      }
-    }
+    const { commentsToEvaluate, reviewCommentsToEvaluate } = this._splitCommentsByPrompt(commentsWithScore);
 
     const relevancesByAI = await this._evaluateComments(
       specificationBody,
@@ -114,7 +90,7 @@ export class ContentEvaluatorModule implements Module {
 
     for (let i = 0; i < commentsWithScore.length; i++) {
       const currentComment = commentsWithScore[i];
-      let currentRelevance = 1; // For comments not in fixed relevance types or missed by OpenAI evaluation
+      let currentRelevance = 1; // For comments not in fixed relevance types and missed by OpenAI evaluation
 
       if (this._fixedRelevances[currentComment.type]) {
         currentRelevance = this._fixedRelevances[currentComment.type];
@@ -133,11 +109,39 @@ export class ContentEvaluatorModule implements Module {
     return commentsWithScore;
   }
 
+  _splitCommentsByPrompt(commentsWithScore: Readonly<GithubCommentScore>[]) {
+    // exclude comments that have fixed relevance multiplier. e.g. review comments = 1
+    const commentsToEvaluate: CommentToEvaluate[] = [];
+    const reviewCommentsToEvaluate: ReviewCommentToEvaluate[] = [];
+    for (let i = 0; i < commentsWithScore.length; i++) {
+      const currentComment = commentsWithScore[i];
+      if (!this._fixedRelevances[currentComment.type]) {
+        if (currentComment.type & CommentKind.PULL) {
+          if (currentComment?.diff_hunk) {
+            //Eval PR comment with diff_hunk, all other PR comments get relevance:1 by default
+
+            reviewCommentsToEvaluate.push({
+              id: currentComment.id,
+              comment: currentComment.content,
+              diff_hunk: currentComment.diff_hunk,
+            });
+          }
+        } else {
+          commentsToEvaluate.push({
+            id: currentComment.id,
+            comment: currentComment.content,
+          });
+        }
+      }
+    }
+    return { commentsToEvaluate, reviewCommentsToEvaluate };
+  }
+
   async _evaluateComments(
     specification: string,
     comments: CommentToEvaluate[],
     reviewComments: ReviewCommentToEvaluate[]
-  ): Promise<Relevances> {
+  ): Promise<RelevancesByOpenAi> {
     let combinedRelevances: Relevances = {};
 
     if (comments.length) {
@@ -176,14 +180,13 @@ export class ContentEvaluatorModule implements Module {
 
     const jsonResponse = JSON.parse(rawResponse);
 
-    const responseType = Type.Record(Type.String(), Type.Number({ minimum: 0, maximum: 1 }));
-
-    if (Value.Check(responseType, jsonResponse)) {
-      const relevances = Value.Convert(responseType, jsonResponse) as { [k: string]: number };
+    try {
+      const relevances = Value.Decode(openAiRelevanceResponseSchema, jsonResponse);
       logger.info(`Relevances by OpenAI: ${JSON.stringify(relevances)}`);
       return relevances;
-    } else {
-      throw new Error(`Invalid response type received from openai while evaluating: ${jsonResponse}`);
+    } catch (e) {
+      logger.error(`Invalid response type received from openai while evaluating: ${jsonResponse} \n\nError: ${e}`);
+      throw new Error("Error in evaluation by OpenAI.");
     }
   }
 

diff --git a/src/types/openai-type.ts b/src/types/openai-type.ts
@@ -0,0 +1,7 @@
+import { Type, Static } from "@sinclair/typebox";
+
+const openAiRelevanceResponseSchema = Type.Record(Type.String(), Type.Number({ minimum: 0, maximum: 1 }));
+
+export type RelevancesByOpenAi = Static<typeof openAiRelevanceResponseSchema>;
+
+export default openAiRelevanceResponseSchema;