From 209db7e2537f694c036ff6e9313ba01559c18ea3 Mon Sep 17 00:00:00 2001
From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com>
Date: Mon, 7 Oct 2024 16:23:10 +0900
Subject: [PATCH] chore: reject on wrong comment evaluation

---
 src/parser/content-evaluator-module.ts |  4 ++--
 tests/process.issue.test.ts            | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts
index e7a6cc9a..7ad2ace2 100644
--- a/src/parser/content-evaluator-module.ts
+++ b/src/parser/content-evaluator-module.ts
@@ -110,8 +110,8 @@ export class ContentEvaluatorModule implements Module {
       prCommentsToEvaluate
     );
 
-    if (Object.keys(relevancesByAi).length !== commentsToEvaluate.length) {
-      console.error("Relevance / Comment length mismatch! \nWill use 1 as relevance for missing comments.");
+    if (Object.keys(relevancesByAi).length !== commentsToEvaluate.length + prCommentsToEvaluate.length) {
+      throw logger.fatal("Relevance / Comment length mistmatch!", { relevancesByAi, commentsToEvaluate });
     }
 
     for (const currentComment of commentsWithScore) {
diff --git a/tests/process.issue.test.ts b/tests/process.issue.test.ts
index 2697c732..78b5f01b 100644
--- a/tests/process.issue.test.ts
+++ b/tests/process.issue.test.ts
@@ -9,7 +9,7 @@ import { DataPurgeModule } from "../src/parser/data-purge-module";
 import { FormattingEvaluatorModule } from "../src/parser/formatting-evaluator-module";
 import { GithubCommentModule } from "../src/parser/github-comment-module";
 import { PermitGenerationModule } from "../src/parser/permit-generation-module";
-import { GithubCommentScore, Processor } from "../src/parser/processor";
+import { Processor } from "../src/parser/processor";
 import { UserExtractorModule } from "../src/parser/user-extractor-module";
 import { parseGitHubUrl } from "../src/start";
 import { db as mockDb } from "./__mocks__/db";
@@ -248,7 +248,7 @@ describe("Modules tests", () => {
     expect(result).toEqual(contentEvaluatorResults);
   });
 
-  it("Should evaluate a failed openai evaluation with relevance 1", async () => {
+  it("Should throw on a failed openai evaluation", async () => {
     jest.spyOn(ContentEvaluatorModule.prototype, "_evaluateComments").mockImplementation(() => {
       return Promise.resolve({});
     });
@@ -260,13 +260,13 @@ describe("Modules tests", () => {
       new FormattingEvaluatorModule(),
       new ContentEvaluatorModule(),
     ];
-    await processor.run(activity);
-    const result = JSON.parse(processor.dump());
-    Object.keys(result).forEach((user) => {
-      expect(result[user]["comments"].length).toBeGreaterThan(0);
-      result[user]["comments"].forEach((comment: GithubCommentScore) => {
-        expect(comment.score?.relevance).toEqual(1);
-      });
+    await expect(processor.run(activity)).rejects.toMatchObject({
+      logMessage: {
+        diff: "```diff\n- Relevance / Comment length mistmatch!\n```",
+        level: "fatal",
+        raw: "Relevance / Comment length mistmatch!",
+        type: "fatal",
+      },
     });
   });