diff --git a/packages/editor/package.json b/packages/editor/package.json index 741cf0141..6d5ea8050 100644 --- a/packages/editor/package.json +++ b/packages/editor/package.json @@ -38,6 +38,8 @@ "@typecell-org/parsers": "^0.0.3", "@typecell-org/frame": "^0.0.3", "@typecell-org/y-penpal": "^0.0.3", + "openai": "^4.11.1", + "ai": "2.2.14", "speakingurl": "^14.0.1", "classnames": "^2.3.1", "fractional-indexing": "^2.0.0", diff --git a/packages/editor/src/app/documentRenderers/richtext/FrameHost.tsx b/packages/editor/src/app/documentRenderers/richtext/FrameHost.tsx index 624082ba4..c9c1d781c 100644 --- a/packages/editor/src/app/documentRenderers/richtext/FrameHost.tsx +++ b/packages/editor/src/app/documentRenderers/richtext/FrameHost.tsx @@ -1,10 +1,11 @@ -import { IframeBridgeMethods } from "@typecell-org/shared"; +import { HostBridgeMethods, IframeBridgeMethods } from "@typecell-org/shared"; import { ContainedElement, useResource } from "@typecell-org/util"; import { PenPalProvider } from "@typecell-org/y-penpal"; import { AsyncMethodReturns, connectToChild } from "penpal"; import { useRef } from "react"; import * as awarenessProtocol from "y-protocols/awareness"; import { parseIdentifier } from "../../../identifiers"; +import { queryOpenAI } from "../../../integrations/ai/openai"; import { DocumentResource } from "../../../store/DocumentResource"; import { DocumentResourceModelProvider } from "../../../store/DocumentResourceModelProvider"; import { SessionStore } from "../../../store/local/SessionStore"; @@ -64,7 +65,7 @@ export function FrameHost(props: { { provider: DocumentResourceModelProvider; forwarder: ModelForwarder } >(); - const methods = { + const methods: HostBridgeMethods = { processYjsMessage: async (message: ArrayBuffer) => { provider.onMessage(message, "penpal"); }, @@ -110,6 +111,7 @@ export function FrameHost(props: { moduleManager.forwarder.dispose(); moduleManagers.delete(identifierStr); }, + queryLLM: queryOpenAI, }; const iframe = document.createElement("iframe"); diff --git a/packages/editor/src/integrations/ai/openai.ts b/packages/editor/src/integrations/ai/openai.ts new file mode 100644 index 000000000..56209a4af --- /dev/null +++ b/packages/editor/src/integrations/ai/openai.ts @@ -0,0 +1,42 @@ +import { OpenAIStream, StreamingTextResponse } from "ai"; +import { ChatCompletionCreateParamsBase, OpenAI } from "openai"; + +export async function queryOpenAI(parameters: { + messages: ChatCompletionCreateParamsBase["messages"]; + functions?: ChatCompletionCreateParamsBase["functions"]; + function_key?: ChatCompletionCreateParamsBase["function_key"]; +}) { + // get key from localstorage + let key = localStorage.getItem("oai-key"); + if (!key) { + key = prompt( + "Please enter your OpenAI key (not shared with TypeCell, stored in your browser):", + ); + if (!key) { + return { + status: "error", + error: "no-key", + } as const; + } + localStorage.setItem("oai-key", key); + } + + const openai = new OpenAI({ + apiKey: key, + dangerouslyAllowBrowser: true, + }); + + const response = await openai.chat.completions.create({ + model: "gpt-4", + stream: true, + ...parameters, + }); + const stream = OpenAIStream(response); + // Respond with the stream + const ret = new StreamingTextResponse(stream); + const data = await ret.text(); + return { + status: "ok", + result: data, + } as const; +} diff --git a/packages/frame/package.json b/packages/frame/package.json index 2d92732cd..b8732d01b 100644 --- a/packages/frame/package.json +++ b/packages/frame/package.json @@ -21,8 +21,6 @@ "mobx": "^6.2.0", "mobx-react-lite": "^3.2.0", "mobx-utils": "^6.0.8", - "openai": "^4.11.1", - "ai": "2.2.14", "prosemirror-model": "^1.19.3", "prosemirror-view": "^1.31.7", "prosemirror-state": "^1.4.3", @@ -48,7 +46,9 @@ "@vitest/coverage-v8": "^0.33.0", "@vitejs/plugin-react": "^4.1.0", "@types/prettier": "^3.0.0", - "chai": "^4.3.7" + "chai": "^4.3.7", + "openai": "^4.11.1", + "ai": "2.2.14" }, "type": "module", "source": "src/index.ts", diff --git a/packages/frame/src/Frame.tsx b/packages/frame/src/Frame.tsx index e8d59f496..f6765d1f6 100644 --- a/packages/frame/src/Frame.tsx +++ b/packages/frame/src/Frame.tsx @@ -302,7 +302,13 @@ export const Frame: React.FC = observer((props) => { execute: async (editor: BlockNoteEditor) => { const p = prompt("AI"); - const commands = await getAICode(p!, tools.newExecutionHost, editor); + const commands = await getAICode( + p!, + tools.newExecutionHost, + editor, + editorStore.current!, + connectionMethods.current!.queryLLM, + ); // debugger; // const commands = [ // { diff --git a/packages/frame/src/ai/ai.ts b/packages/frame/src/ai/ai.ts index 9b24917b4..75f21c52e 100644 --- a/packages/frame/src/ai/ai.ts +++ b/packages/frame/src/ai/ai.ts @@ -1,12 +1,13 @@ // import LocalExecutionHost from "../../../runtime/executor/executionHosts/local/LocalExecutionHost" import "@blocknote/core/style.css"; -import { OpenAIStream, StreamingTextResponse } from "ai"; import * as mobx from "mobx"; import * as monaco from "monaco-editor"; -import { OpenAI } from "openai"; +import { ChatCompletionMessageParam } from "openai"; import { BlockNoteEditor } from "@blocknote/core"; +import { HostBridgeMethods } from "@typecell-org/shared"; import { uri } from "vscode-lib"; +import { EditorStore } from "../EditorStore"; import { compile } from "../runtime/compiler/compilers/MonacoCompiler"; import { ExecutionHost } from "../runtime/executor/executionHosts/ExecutionHost"; import { customStringify } from "../stringify"; @@ -114,15 +115,12 @@ but instead: $.complexObject.newProperty = 5; `; -const openai = new OpenAI({ - apiKey: "", - dangerouslyAllowBrowser: true, -}); - export async function getAICode( prompt: string, executionHost: ExecutionHost, editor: BlockNoteEditor, + editorStore: EditorStore, + queryLLM: HostBridgeMethods["queryLLM"], ) { const models = monaco.editor.getModels(); const typeCellModels = models.filter((m) => @@ -130,19 +128,48 @@ export async function getAICode( ); const blocks = editor.topLevelBlocks; + let blockContexts: any[] = []; + const iterateBlocks = (blocks: any[]) => { + for (const block of blocks) { + const b = editorStore.getBlock(block.id); + if (b?.context?.default) { + blockContexts.push(b.context.default); + } + iterateBlocks(block.children); + } + }; + iterateBlocks(blocks); + + blockContexts = blockContexts.map((output) => + Object.fromEntries( + Object.getOwnPropertyNames(output).map((key) => [ + key, + mobx.toJS(output[key]), + ]), + ), + ); + const tmpModel = monaco.editor.createModel( "", "typescript", uri.URI.parse("file:///tmp.tsx"), ); - tmpModel.setValue(`import React from "!typecell:typecell.org/dqBLFEyFuSUu1"; - import * as $ from "!typecell:typecell.org/dqBLFEyFuSUu1"; + tmpModel.setValue(`import * as React from "react"; + import * as $ from "!typecell:typecell.org/dVeeYvbKcq2Nz"; // expands object types one level deep type Expand = T extends infer O ? { [K in keyof O]: O[K] extends { Key: React.Key | null } ? "[REACT]" : O[K] } : never; // expands object types recursively type ExpandRecursively = T extends object - ? T extends infer O ? { [K in keyof O]: O[K] extends { key: React.Key } ? "[REACT ELEMENT]" : ExpandRecursively } : never + ? T extends (...args: any[]) => any + ? T + : T extends infer O + ? { + [K in keyof O]: O[K] extends { key: React.Key } + ? "[REACT ELEMENT]" + : ExpandRecursively; + } + : never : T; // ? T extends infer O ? { [K in keyof O]: ExpandRecursively } : never @@ -158,7 +185,6 @@ type ExpandRecursively = T extends object const def2 = await ts.getQuickInfoAtPosition(tmpModel.uri.toString(), pos); const contextType = def2.displayParts.map((x: any) => x.text).join(""); // const def3 = await ts.get(tmpModel.uri.toString(), pos, {}); - tmpModel.dispose(); const codeInfoPromises = typeCellModels.map(async (m) => { @@ -212,7 +238,9 @@ type ExpandRecursively = T extends object if (block.children) { block.children = block.children.map(cleanBlock); } - block.content = block.content.map((x: any) => x.text).join(""); + if (Array.isArray(block.content)) { + block.content = block.content.map((x: any) => x.text).join(""); + } return block; } // console.log("request", JSON.stringify(blocks).length); @@ -225,48 +253,60 @@ type ExpandRecursively = T extends object contextType.replace("type ContextType = ", "const $: ") + " = " + JSON.stringify(outputJS); - // Ask OpenAI for a streaming chat completion given the prompt - const response = await openai.chat.completions.create({ - // model: "gpt-3.5-turbo-16k", - model: "gpt-4", - stream: true, - messages: [ - { - role: "system", - content: TYPECELL_PROMPT, - }, - { - role: "user", - content: `This is my document data: + + const blockContextInfo = blockContexts.length + ? `typecell.editor.findBlocks = (predicate: (context) => boolean) { + return (${JSON.stringify(blockContexts)}).find(predicate); + }` + : undefined; + + const messages: Array = [ + { + role: "system", + content: TYPECELL_PROMPT, + }, + { + role: "user", + content: `This is my document data: """${JSON.stringify(sanitized)}"""`, - }, - { - role: "user", - content: - "This is the type and runtime data available under the reactive $ variable for read / write access. If you need to change / read some information from the live document, it's likely you need to access it from here using $. \n" + - contextInfo, - }, - // codeInfos.length - // ? { - // role: "user", - // content: `This is the runtime / compiler data of the Code Blocks (CodeBlockRuntimeInfo[]): - // """${JSON.stringify(codeInfos)}"""`, - // } - // : { - // role: "user", - // content: `There are no code blocks in the document, so there's no runtime / compiler data for these (CodeBlockRuntimeInfo[]).`, - // }, - { - role: "system", - content: `You are an AI assistant helping user to modify his document. This means changes can either be code related (in that case, you'll need to add / modify Code Blocks), + }, + { + role: "user", + content: + "This is the type and runtime data available under the reactive $ variable for read / write access. If you need to change / read some information from the live document, it's likely you need to access it from here using $. \n" + + contextInfo + + (blockContextInfo + ? "\n" + + `We also have this function "typecell.editor.findBlocks" to extract runtime data from blocks \n` + + blockContextInfo + : ""), + }, + + // codeInfos.length + // ? { + // role: "user", + // content: `This is the runtime / compiler data of the Code Blocks (CodeBlockRuntimeInfo[]): + // """${JSON.stringify(codeInfos)}"""`, + // } + // : { + // role: "user", + // content: `There are no code blocks in the document, so there's no runtime / compiler data for these (CodeBlockRuntimeInfo[]).`, + // }, + { + role: "system", + content: `You are an AI assistant helping user to modify his document. This means changes can either be code related (in that case, you'll need to add / modify Code Blocks), or not at all (in which case you'll need to add / modify regular blocks), or a mix of both.`, - }, - { - role: "user", - content: prompt, // + - // " . \n\nRemember to reply ONLY with OperationsResponse JSON (DO NOT add any further comments). So start with [{ and end with }]", - }, - ], + }, + { + role: "user", + content: prompt, // + + // " . \n\nRemember to reply ONLY with OperationsResponse JSON (DO NOT add any further comments). So start with [{ and end with }]", + }, + ]; + + // Ask OpenAI for a streaming chat completion given the prompt + const response = await queryLLM({ + messages, functions: [ { name: "updateDocument", @@ -372,12 +412,13 @@ type ExpandRecursively = T extends object }, }); - const stream = OpenAIStream(response); - - // Respond with the stream - const ret = new StreamingTextResponse(stream); - const data = await ret.json(); - console.log(data); + console.log(messages); - return JSON.parse(data.function_call.arguments).operations; + if (response.status === "ok") { + const data = JSON.parse(response.result); + return JSON.parse(data.function_call.arguments).operations; + } else { + console.error("queryLLM error", response.error); + } + return undefined; } diff --git a/packages/frame/src/ai/applyChanges.ts b/packages/frame/src/ai/applyChanges.ts index 79ab87ec6..c804e3d52 100644 --- a/packages/frame/src/ai/applyChanges.ts +++ b/packages/frame/src/ai/applyChanges.ts @@ -1,4 +1,5 @@ import { error, uniqueId } from "@typecell-org/util"; +import * as ypm from "y-prosemirror"; import { Awareness } from "y-protocols/awareness"; import * as Y from "yjs"; import { BlockOperation, OperationsResponse } from "./types"; @@ -92,6 +93,9 @@ export async function applyChange( data: Y.XmlFragment, awareness: Awareness, ) { + const transact = (op: () => void) => { + Y.transact(data.doc!, op, ypm.ySyncPluginKey); + }; if (op.type === "add") { const node = findBlock(op.afterId, data); if (!node) { @@ -104,8 +108,9 @@ export async function applyChange( child.insert(0, [yText]); newElement.insert(0, [child]); // TODO: create block - (node.parent as Y.XmlElement).insertAfter(node, [newElement]); - + transact(() => { + (node.parent as Y.XmlElement).insertAfter(node, [newElement]); + }); // start typing text content for (let i = 0; i < op.content.length; i++) { const start = Y.createRelativePositionFromTypeIndex(yText, i); @@ -113,7 +118,9 @@ export async function applyChange( updateState(awareness, start, end); // return new RelativeSelection(start, end, sel.getDirection()) - yText.insert(i, op.content[i]); + transact(() => { + yText.insert(i, op.content[i]); + }); await new Promise((resolve) => setTimeout(resolve, 20)); } } else if (op.type === "delete") { @@ -131,7 +138,9 @@ export async function applyChange( await new Promise((resolve) => setTimeout(resolve, 200)); - (node.parent as Y.XmlElement).delete(findParentIndex(node), 1); + transact(() => { + (node.parent as Y.XmlElement).delete(findParentIndex(node), 1); + }); await new Promise((resolve) => setTimeout(resolve, 20)); } else if (op.type === "update") { const node = findBlock(op.id, data); @@ -160,7 +169,9 @@ export async function applyChange( updateState(awareness, start, end); // return new RelativeSelection(start, end, sel.getDirection()) - yText.insert(step.from + i, step.text[i]); + transact(() => { + yText.insert(step.from + i, step.text[i]); + }); await new Promise((resolve) => setTimeout(resolve, 20)); } // cell.code.delete(step.from, step.length); @@ -172,7 +183,9 @@ export async function applyChange( ); updateState(awareness, start, end); await new Promise((resolve) => setTimeout(resolve, 200)); - yText.delete(step.from, step.length); + transact(() => { + yText.delete(step.from, step.length); + }); await new Promise((resolve) => setTimeout(resolve, 20)); } } diff --git a/packages/frame/src/codeblocks/MonacoCodeBlock.tsx b/packages/frame/src/codeblocks/MonacoCodeBlock.tsx index 1b70379c7..affe60eb3 100644 --- a/packages/frame/src/codeblocks/MonacoCodeBlock.tsx +++ b/packages/frame/src/codeblocks/MonacoCodeBlock.tsx @@ -85,6 +85,7 @@ export const MonacoCodeBlock = createTipTapBlock<"codeblock", any>({ // class: styles.blockContent, "data-content-type": this.name, }), + 0, ]; }, diff --git a/packages/frame/src/runtime/executor/components/ModelOutput.ts b/packages/frame/src/runtime/executor/components/ModelOutput.ts index 816b9a11a..fef85ffdf 100644 --- a/packages/frame/src/runtime/executor/components/ModelOutput.ts +++ b/packages/frame/src/runtime/executor/components/ModelOutput.ts @@ -11,7 +11,9 @@ export class ModelOutput extends lifecycle.Disposable { private autorunDisposer: (() => void) | undefined; public value: any = undefined; - public _defaultValue: any = {}; + public _defaultValue = { + value: {} as any, + }; public typeVisualizers = observable.map< string, { @@ -21,6 +23,7 @@ export class ModelOutput extends lifecycle.Disposable { constructor(private context: any) { super(); + makeObservable(this, { typeVisualizers: observable.ref, value: observable.ref, @@ -70,13 +73,14 @@ export class ModelOutput extends lifecycle.Disposable { } } - this._defaultValue = newValue.default; + // hacky nesting to make sure our customAnnotation (for react elements) is used + this._defaultValue = { value: newValue.default }; if (changed) { if (Object.hasOwn(newValue, "default")) { Object.defineProperty(newValue, "default", { get: () => { - return this.defaultValue; + return this.defaultValue.value; }, }); } diff --git a/packages/frame/src/runtime/executor/lib/exports.tsx b/packages/frame/src/runtime/executor/lib/exports.tsx index dc0ade12c..7bbf7a83e 100644 --- a/packages/frame/src/runtime/executor/lib/exports.tsx +++ b/packages/frame/src/runtime/executor/lib/exports.tsx @@ -4,7 +4,7 @@ import memoize from "lodash.memoize"; import { autorun, comparer, computed, runInAction } from "mobx"; import { observer } from "mobx-react-lite"; import { computedFn, createTransformer } from "mobx-utils"; -import React, { useEffect, useMemo } from "react"; +import { useEffect, useMemo } from "react"; import { EditorStore } from "../../../EditorStore"; import { AutoForm, AutoFormProps } from "./autoForm"; import { Input } from "./input/Input"; @@ -125,7 +125,7 @@ export default function getExposeGlobalVariables( }, }; return { - memoize: (func: (...args: any[]) => any) => { + memoize: any>(func: T): T => { const wrapped = async function (this: any, ...args: any[]) { const ret = await func.apply(this, args); // if (typeof ret === "object") { @@ -135,7 +135,7 @@ export default function getExposeGlobalVariables( }; return memoize(wrapped, (args) => { return JSON.stringify(args); - }); + }) as any as T; }, // routing, // // DocumentView, @@ -195,6 +195,7 @@ export default function getExposeGlobalVariables( return undefined; } return autorun(() => { + // console.log("autorun setting", func); func(); }); }, diff --git a/packages/shared/src/frameInterop/HostBridgeMethods.ts b/packages/shared/src/frameInterop/HostBridgeMethods.ts index 6c2919c11..2364dfb39 100644 --- a/packages/shared/src/frameInterop/HostBridgeMethods.ts +++ b/packages/shared/src/frameInterop/HostBridgeMethods.ts @@ -18,4 +18,22 @@ export type HostBridgeMethods = { * Function for y-penpal */ processYjsMessage: (message: Uint8Array) => Promise; + + queryLLM: (parameters: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messages: any[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + functions?: any[]; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + function_call?: any; + }) => Promise< + | { + status: "ok"; + result: string; + } + | { + status: "error"; + error: string; + } + >; };