chore: reject on wrong comment evaluation

gentlementlegen · Oct 7, 2024 · 209db7e · 209db7e
1 parent a08e6f4
commit 209db7e
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 11 deletions.
diff --git a/src/parser/content-evaluator-module.ts b/src/parser/content-evaluator-module.ts
@@ -110,8 +110,8 @@ export class ContentEvaluatorModule implements Module {
       prCommentsToEvaluate
     );
 
-    if (Object.keys(relevancesByAi).length !== commentsToEvaluate.length) {
-      console.error("Relevance / Comment length mismatch! \nWill use 1 as relevance for missing comments.");
+    if (Object.keys(relevancesByAi).length !== commentsToEvaluate.length + prCommentsToEvaluate.length) {
+      throw logger.fatal("Relevance / Comment length mistmatch!", { relevancesByAi, commentsToEvaluate });
     }
 
     for (const currentComment of commentsWithScore) {

diff --git a/tests/process.issue.test.ts b/tests/process.issue.test.ts
@@ -9,7 +9,7 @@ import { DataPurgeModule } from "../src/parser/data-purge-module";
 import { FormattingEvaluatorModule } from "../src/parser/formatting-evaluator-module";
 import { GithubCommentModule } from "../src/parser/github-comment-module";
 import { PermitGenerationModule } from "../src/parser/permit-generation-module";
-import { GithubCommentScore, Processor } from "../src/parser/processor";
+import { Processor } from "../src/parser/processor";
 import { UserExtractorModule } from "../src/parser/user-extractor-module";
 import { parseGitHubUrl } from "../src/start";
 import { db as mockDb } from "./__mocks__/db";
@@ -248,7 +248,7 @@ describe("Modules tests", () => {
     expect(result).toEqual(contentEvaluatorResults);
   });
 
-  it("Should evaluate a failed openai evaluation with relevance 1", async () => {
+  it("Should throw on a failed openai evaluation", async () => {
     jest.spyOn(ContentEvaluatorModule.prototype, "_evaluateComments").mockImplementation(() => {
       return Promise.resolve({});
     });
@@ -260,13 +260,13 @@ describe("Modules tests", () => {
       new FormattingEvaluatorModule(),
       new ContentEvaluatorModule(),
     ];
-    await processor.run(activity);
-    const result = JSON.parse(processor.dump());
-    Object.keys(result).forEach((user) => {
-      expect(result[user]["comments"].length).toBeGreaterThan(0);
-      result[user]["comments"].forEach((comment: GithubCommentScore) => {
-        expect(comment.score?.relevance).toEqual(1);
-      });
+    await expect(processor.run(activity)).rejects.toMatchObject({
+      logMessage: {
+        diff: "```diff\n- Relevance / Comment length mistmatch!\n```",
+        level: "fatal",
+        raw: "Relevance / Comment length mistmatch!",
+        type: "fatal",
+      },
     });
   });