basic AI integration

packages/editor/package.json

@@ -38,6 +38,8 @@
     "@typecell-org/parsers": "^0.0.3",
     "@typecell-org/frame": "^0.0.3",
     "@typecell-org/y-penpal": "^0.0.3",
+    "openai": "^4.11.1",
+    "ai": "2.2.14",
     "speakingurl": "^14.0.1",
     "classnames": "^2.3.1",
     "fractional-indexing": "^2.0.0",

packages/editor/src/app/documentRenderers/richtext/FrameHost.tsx

@@ -1,10 +1,11 @@
-import { IframeBridgeMethods } from "@typecell-org/shared";
+import { HostBridgeMethods, IframeBridgeMethods } from "@typecell-org/shared";
 import { ContainedElement, useResource } from "@typecell-org/util";
 import { PenPalProvider } from "@typecell-org/y-penpal";
 import { AsyncMethodReturns, connectToChild } from "penpal";
 import { useRef } from "react";
 import * as awarenessProtocol from "y-protocols/awareness";
 import { parseIdentifier } from "../../../identifiers";
+import { queryOpenAI } from "../../../integrations/ai/openai";
 import { DocumentResource } from "../../../store/DocumentResource";
 import { DocumentResourceModelProvider } from "../../../store/DocumentResourceModelProvider";
 import { SessionStore } from "../../../store/local/SessionStore";
@@ -64,7 +65,7 @@ export function FrameHost(props: {
       { provider: DocumentResourceModelProvider; forwarder: ModelForwarder }
     >();
 
-    const methods = {
+    const methods: HostBridgeMethods = {
       processYjsMessage: async (message: ArrayBuffer) => {
         provider.onMessage(message, "penpal");
       },
@@ -110,6 +111,7 @@ export function FrameHost(props: {
         moduleManager.forwarder.dispose();
         moduleManagers.delete(identifierStr);
       },
+      queryLLM: queryOpenAI,
     };
 
     const iframe = document.createElement("iframe");

packages/editor/src/integrations/ai/openai.ts

@@ -0,0 +1,43 @@
+import { OpenAIStream, StreamingTextResponse } from "ai";
+import { OpenAI } from "openai";
+
+export async function queryOpenAI(parameters: {
+  messages: OpenAI.Chat.ChatCompletionCreateParams["messages"];
+  functions?: OpenAI.Chat.ChatCompletionCreateParams["functions"];
+  function_call?: OpenAI.Chat.ChatCompletionCreateParams["function_call"];
+}) {
+  // get key from localstorage
+  let key = localStorage.getItem("oai-key");
+  if (!key) {
+    key = prompt(
+      "Please enter your OpenAI key (not shared with TypeCell, stored in your browser):",
+    );
+    if (!key) {
+      return {
+        status: "error",
+        error: "no-key",
+      } as const;
+    }
+    localStorage.setItem("oai-key", key);
+  }
+
+  const openai = new OpenAI({
+    apiKey: key,
+    // this should be ok as we are not exposing any keys
+    dangerouslyAllowBrowser: true,
+  });
+
+  const response = await openai.chat.completions.create({
+    model: "gpt-4",
+    stream: true,
+    ...parameters,
+  });
+  const stream = OpenAIStream(response);
+  // Respond with the stream
+  const ret = new StreamingTextResponse(stream);
+  const data = await ret.text();
+  return {
+    status: "ok",
+    result: data,
+  } as const;
+}

packages/engine/src/executor.ts

@@ -13,6 +13,7 @@ async function resolveDependencyArray(
   userDisposes: Array<() => void>,
 ) {
   const runContext = {
+    context,
     onDispose: (disposer: () => void) => {
       userDisposes.push(() => {
         try {
@@ -47,6 +48,7 @@ async function resolveDependencyArray(
 }
 
 export type RunContext = {
+  context: TypeCellContext<any>;
   onDispose: (disposer: () => void) => void;
 };
 
@@ -120,6 +122,7 @@ export async function runModule(
       disposeEveryRun.push(hooks.disposeAll);
       let executionPromise: Promise<any>;
       try {
+        console.log("execute", mod.factoryFunction + "");
         executionPromise = mod.factoryFunction.apply(
           undefined,
           argsToCallFunctionWith,

packages/engine/src/modules.ts

@@ -28,10 +28,11 @@ export function getModulesFromWrappedPatchedTypeCellFunction(
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   caller: () => any,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  scope: any
+  scope: any,
 ): Module[] {
   const modules: Module[] = [];
   const define = createDefine(modules);
+  console.log("evaluate (module)", caller + "");
   caller.apply({ ...scope, define });
   return modules;
 }
@@ -43,10 +44,11 @@ export function getModulesFromWrappedPatchedTypeCellFunction(
 export function getModulesFromPatchedTypeCellCode(
   code: string,
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  scope: any
+  scope: any,
 ): Module[] {
   const modules: Module[] = [];
   const define = createDefine(modules);
+  console.log("evaluate (module)", code);
   // eslint-disable-next-line
   const f = new Function(code);
   f.apply({ ...scope, define });
@@ -57,7 +59,7 @@ function createDefine(modules: Module[]) {
   return function typeCellDefine(
     moduleNameOrDependencyArray: string | string[],
     dependencyArrayOrFactoryFunction: string[] | Function,
-    factoryFunction?: Function
+    factoryFunction?: Function,
   ) {
     const moduleName: string | typeof unnamedModule =
       typeof moduleNameOrDependencyArray === "string"
@@ -118,7 +120,7 @@ export function getPatchedTypeCellCode(compiledCode: string, scope: any) {
 
   totalCode = totalCode.replace(
     /^\s*(define\((".*", )?\[.*\], )function/gm,
-    "$1async function"
+    "$1async function",
   ); // TODO: remove await?
 
   return totalCode;

packages/frame/package.json

@@ -3,6 +3,7 @@
   "version": "0.0.3",
   "private": true,
   "dependencies": {
+    "@atlaskit/form": "^8.11.8",
     "@blocknote/core": "^0.9.3",
     "@blocknote/react": "^0.9.3",
     "@typecell-org/util": "^0.0.3",
@@ -14,12 +15,12 @@
     "@floating-ui/react": "^0.25.1",
     "@syncedstore/yjs-reactive-bindings": "^0.5.1",
     "lodash.memoize": "^4.1.2",
-    "mobx-utils": "^6.0.8",
     "localforage": "^1.10.0",
     "lz-string": "^1.4.4",
     "monaco-editor": "^0.35.0",
     "mobx": "^6.2.0",
     "mobx-react-lite": "^3.2.0",
+    "mobx-utils": "^6.0.8",
     "prosemirror-model": "^1.19.3",
     "prosemirror-view": "^1.31.7",
     "prosemirror-state": "^1.4.3",
@@ -33,7 +34,8 @@
     "typescript": "5.0.4",
     "vscode-lib": "^0.1.2",
     "y-protocols": "^1.0.5",
-    "yjs": "^13.6.4"
+    "yjs": "^13.6.4",
+    "y-prosemirror": "^1.0.20"
   },
   "devDependencies": {
     "cross-fetch": "^4.0.0",
@@ -45,7 +47,9 @@
     "@vitest/coverage-v8": "^0.33.0",
     "@vitejs/plugin-react": "^4.1.0",
     "@types/prettier": "^3.0.0",
-    "chai": "^4.3.7"
+    "chai": "^4.3.7",
+    "openai": "^4.11.1",
+    "ai": "2.2.14"
   },
   "type": "module",
   "source": "src/index.ts",

packages/frame/src/EditorStore.ts

@@ -21,36 +21,40 @@
   public editor: BlockNoteEditor | undefined;
   /** @internal */
   public executionHost: LocalExecutionHost | undefined;
   public topLevelBlocks: any;
 
+  public readonly customBlocks = new Map<string, any>();
+  public readonly blockSettings = new Map<string, any>();
+
   constructor() {
     makeObservable(this, {
       customBlocks: observable.shallow,
-      add: action,
-      delete: action,
+      addCustomBlock: action,
+      deleteCustomBlock: action,
+      blockSettings: observable.shallow,
+      addBlockSettings: action,
+      deleteBlockSettings: action,
       topLevelBlocks: observable.ref,
     });
 
     onBecomeObserved(this, "topLevelBlocks", () => {
       this.editor!.onEditorContentChange(() => {
         runInAction(() => {
           this.topLevelBlocks = this.editor!.topLevelBlocks.map((block) =>
             this.getBlock(block.id),
           );
         });
       });
       this.topLevelBlocks = this.editor!.topLevelBlocks.map((block) =>
         this.getBlock(block.id),
       );
     });
   }
 
-  customBlocks = new Map<string, any>();
-
   /**
    * Add a custom block (slash menu command) to the editor
    */
-  public add(config: any) {
+  public addCustomBlock(config: any) {
     if (this.customBlocks.has(config.id)) {
       // already has block with this id, maybe loop of documents?
       return;
@@ -61,10 +65,28 @@
   /**
    * Remove a custom block (slash menu command) from the editor
    */
-  public delete(config: any) {
+  public deleteCustomBlock(config: any) {
     this.customBlocks.delete(config.id);
   }
 
+  /**
+   * Add a block settings (block settings menu) to the editor
+   */
+  public addBlockSettings(config: any) {
+    if (this.blockSettings.has(config.id)) {
+      // already has block with this id, maybe loop of documents?
+      return;
+    }
+    this.blockSettings.set(config.id, config);
+  }
+
+  /**
+   * Remove block settings (block settings menu) from the editor
+   */
+  public deleteBlockSettings(config: any) {
+    this.blockSettings.delete(config.id);
+  }
+
   /**
    * EXPERIMENTAL
    * @internal
@@ -180,11 +202,11 @@
       runInAction(() => {
         this.block = newBlock;
 
-        if (newBlock.props.storage !== JSON.stringify(this.storage)) {
-          if (newBlock.props.storage) {
+        if ((newBlock.props as any).storage !== JSON.stringify(this.storage)) {
+          if (newBlock.props as any) {
             try {
               console.log("update cell storage");
-              this.storage = JSON.parse(newBlock.props.storage) || {};
+              this.storage = JSON.parse((newBlock.props as any).storage) || {};
             } catch (e) {
               console.error(e);
             }
@@ -221,7 +243,7 @@
           editor.updateBlock(this.block, {
             props: {
               storage: val,
-            },
+            } as any,
           });
         }
       },