validate code examples for threads (#155)

package.json

@@ -16,6 +16,7 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@effect/language-service": "^0.1.0",
     "@octokit/types": "^13.4.1",
     "@types/node": "^20.12.7",
     "tsx": "^4.7.2",

src/AutoThreads.ts

@@ -1,6 +1,6 @@
 import { Schema, TreeFormatter } from "@effect/schema"
 import { ChannelsCache } from "bot/ChannelsCache"
-import { OpenAI, OpenAIError } from "bot/OpenAI"
+import { OpenAI, OpenAIError, OpenAITool } from "bot/OpenAI"
 import { LayerUtils } from "bot/_common"
 import * as Str from "bot/utils/String"
 import { Discord, DiscordREST, Ix, Perms, UI } from "dfx"
@@ -11,6 +11,7 @@ import {
 } from "dfx/gateway"
 import {
   Cause,
+  Console,
   Context,
   Data,
   Duration,
@@ -23,9 +24,6 @@ import {
 
 const retryPolicy = pipe(
   Schedule.fixed(Duration.millis(500)),
-  Schedule.whileInput(
-    (_: OpenAIError | Cause.NoSuchElementException) => _._tag === "OpenAIError",
-  ),
   Schedule.intersect(Schedule.recurs(2)),
 )
 
@@ -40,6 +38,28 @@ export class PermissionsError extends Data.TaggedError("PermissionsError")<{
   readonly subject: string
 }> {}
 
+class MessageInfo extends Schema.Class<MessageInfo>("MessageInfo")({
+  short_title: Schema.String.pipe(
+    Schema.description("A short title summarizing the message"),
+  ),
+  has_code_examples: Schema.Boolean.pipe(
+    Schema.description("Are there code examples in the message?"),
+  ),
+  has_code_fences: Schema.Boolean.pipe(
+    Schema.description("Does the message contain code fences, e.g. ```"),
+  ),
+}) {
+  get missingCodeFences() {
+    return this.has_code_examples && !this.has_code_fences
+  }
+}
+
+const messageInfo = new OpenAITool(
+  "message_info",
+  "Extract a short title and validate a message",
+  MessageInfo,
+)
+
 const make = ({ topicKeyword }: { readonly topicKeyword: string }) =>
   Effect.gen(function* (_) {
     const openai = yield* _(OpenAI)
@@ -70,33 +90,36 @@ const make = ({ topicKeyword }: { readonly topicKeyword: string }) =>
           .get(message.guild_id!, message.channel_id)
           .pipe(Effect.flatMap(EligibleChannel)),
       }).pipe(
-        Effect.bind("title", () =>
-          pipe(
-            Str.nonEmpty(message.content),
-            Effect.flatMap(content =>
-              pipe(
-                openai.generateTitle(content),
-                Effect.retry(retryPolicy),
-                Effect.tapErrorCause(_ => Effect.log(_)),
-              ),
-            ),
+        Effect.bind("info", () =>
+          openai.fn(messageInfo, message.content).pipe(
+            Effect.retry({
+              schedule: retryPolicy,
+              while: err => err._tag === "OpenAIError",
+            }),
+            Effect.tapErrorCause(_ => Effect.log(_)),
             Effect.orElseSucceed(() =>
               pipe(
                 Option.fromNullable(message.member?.nick),
                 Option.getOrElse(() => message.author.username),
-                _ => `${_}'s thread`,
+                _ =>
+                  new MessageInfo({
+                    short_title: `${_}'s thread`,
+                    has_code_examples: false,
+                    has_code_fences: false,
+                  }),
               ),
             ),
           ),
         ),
-        Effect.flatMap(
-          ({ channel, title }) =>
+        Effect.bind(
+          "thread",
+          ({ channel, info }) =>
             rest.startThreadFromMessage(channel.id, message.id, {
-              name: Str.truncate(title, 100),
+              name: Str.truncate(info.short_title, 100),
               auto_archive_duration: 1440,
             }).json,
         ),
-        Effect.flatMap(thread =>
+        Effect.tap(({ thread }) =>
           rest.createMessage(thread.id, {
             components: UI.grid([
               [
@@ -113,6 +136,20 @@ const make = ({ topicKeyword }: { readonly topicKeyword: string }) =>
             ]),
           }),
         ),
+        Effect.tap(({ thread, info }) =>
+          rest
+            .createMessage(thread.id, {
+              content: `It looks like your code examples might be missing code fences.
+
+You can wrap your code like this:
+
+${"\\`\\`\\`"}ts
+const a = 123
+${"\\`\\`\\`"}
+`,
+            })
+            .pipe(Effect.when(() => info.missingCodeFences)),
+        ),
         Effect.withSpan("AutoThreads.handleMessages"),
         Effect.catchTags({
           ParseError: error =>

src/OpenAI.ts

@@ -1,15 +1,16 @@
+import { JSONSchema, Schema } from "@effect/schema"
 import { LayerUtils } from "bot/_common"
 import * as Str from "bot/utils/String"
 import {
-  Secret,
   Context,
   Data,
   Effect,
   Layer,
+  Metric,
   Option,
-  pipe,
   Predicate,
-  Metric,
+  Secret,
+  pipe,
 } from "effect"
 import * as Tokenizer from "gpt-tokenizer"
 import * as OAI from "openai"
@@ -41,6 +42,68 @@ export interface Message {
   readonly content: string
 }
 
+export interface ChoiceToolCall<A>
+  extends Schema.Struct<{
+    message: Schema.Struct<{
+      tool_calls: Schema.NonEmptyArray<
+        Schema.Struct<{
+          function: Schema.Struct<{
+            name: Schema.Literal<[string]>
+            arguments: Schema.Schema<A, string, never>
+          }>
+        }>
+      >
+    }>
+  }> {}
+
+const ChoiceToolCall = <A, I>(
+  name: string,
+  schema: Schema.Schema<A, I>,
+): ChoiceToolCall<A> =>
+  Schema.Struct({
+    message: Schema.Struct({
+      tool_calls: Schema.NonEmptyArray(
+        Schema.Struct({
+          function: Schema.Struct({
+            name: Schema.Literal(name),
+            arguments: Schema.parseJson(schema),
+          }),
+        }),
+      ),
+    }),
+  })
+
+export class OpenAITool<A> {
+  constructor(
+    readonly name: string,
+    readonly description: string,
+    readonly schema: Schema.Schema<A, any> & {
+      readonly fields: Record<string, Schema.Schema<any, any>>
+    },
+  ) {
+    this.jsonSchema = JSONSchema.make(Schema.Struct(schema.fields)) as any
+    this.choiceSchema = ChoiceToolCall(name, schema)
+  }
+
+  readonly jsonSchema: Record<string, unknown>
+  readonly choiceSchema: ChoiceToolCall<A>
+
+  decodeChoice(value: OAI.OpenAI.ChatCompletion.Choice | undefined) {
+    return Schema.decodeUnknown(this.choiceSchema)(value)
+  }
+
+  get tool(): OAI.OpenAI.ChatCompletionTool {
+    return {
+      type: "function",
+      function: {
+        name: this.name,
+        description: this.description,
+        parameters: this.jsonSchema,
+      },
+    }
+  }
+}
+
 const make = (params: {
   readonly apiKey: Secret.Secret
   readonly organization: Option.Option<Secret.Secret>
@@ -62,6 +125,32 @@ const make = (params: {
       Effect.withSpan("OpenAI.call"),
     )
 
+  const fn = <A>(tool: OpenAITool<A>, prompt: string) =>
+    call((_, signal) =>
+      _.chat.completions.create(
+        {
+          model: "gpt-4-turbo-preview",
+          tools: [tool.tool],
+          tool_choice: {
+            type: "function",
+            function: {
+              name: tool.name,
+            },
+          },
+          messages: [
+            {
+              role: "user",
+              content: prompt,
+            },
+          ],
+        },
+        { signal },
+      ),
+    ).pipe(
+      Effect.andThen(_ => tool.decodeChoice(_.choices[0])),
+      Effect.map(_ => _.message.tool_calls[0].function.arguments),
+    )
+
   const generateTitle = (prompt: string) =>
     call((_, signal) =>
       _.chat.completions.create(
@@ -185,6 +274,7 @@ The title of this chat is "${title}".`,
   return {
     client,
     call,
+    fn,
     generateTitle,
     generateReply,
     generateDocs,

tsconfig.json

@@ -18,7 +18,8 @@
       "bot": ["./src/index.js"],
       "bot/*": ["./src/*.js"]
     },
-    "tsBuildInfoFile": "./tsconfig.tsbuildinfo"
+    "tsBuildInfoFile": "./tsconfig.tsbuildinfo",
+    "plugins": [{ "name": "@effect/langauge-service" }]
   },
   "include": ["src/**/*"]
 }