refactor: convert to turborepo monorepo structure (#286)

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: ben <ben@prologe.io>

.github/workflows/build.yml

@@ -14,8 +14,13 @@ jobs:
         with:
           node-version: "18.x"
 
+      - name: Install pnpm
+        uses: pnpm/action-setup@v2
+        with:
+          version: 8
+
       - name: Install dependencies
-        run: npm install
+        run: pnpm install
 
       - name: Build plugin
-        run: npm run build
+        run: pnpm build

.gitignore

@@ -1,5 +1,9 @@
 # vscode
 
+.turbo/*
+
+*/.turbo/*
+
 .vscode 
 
 # Intellij

README.md

@@ -1,9 +1,3 @@
-
-
-
-
-
-
 # File Organizer 2000
 
 Automate the most painful part of your Obsidian worfklow.
@@ -104,25 +98,47 @@ Choose between the setups below:
    For Linux/macOS:
 
    ```sh
-   cd web && npm run build:self-host &&  npm run start
+   cd packages/web && pnpm build:self-host && pnpm start
    ```
 
-   And make sure you have your  `OPENAI_API_KEY` variable set up in your `.env.local` file inside the app root folder.
+   And make sure you have your `OPENAI_API_KEY` variable set up in your `.env.local` file inside the `packages/web` directory.
 
    For Windows (PowerShell):
 
    ```sh
-   cd web; npm run build:self-host; npm run start
-    ```
+   cd packages/web; pnpm build:self-host; pnpm start
+   ```
+
+   And make sure you have your `OPENAI_API_KEY` variable set up in your `.env.local` file inside the `packages/web` directory.
 
-   And make sure you have your  `OPENAI_API_KEY` variable set up in your `.env.local` file inside the app root folder.
+2. Enable self-hosted mode in plugin settings:
 
+   <img width="707" alt="Screenshot 2024-04-13 at 07 16 21" src="https://github.com/different-ai/file-organizer-2000/assets/11430621/ca2222c9-cb8d-4d15-8459-2da4c9662f24">
 
-2. Go inside the Settings of the plugin and enable "Self-hosted"
+### C. Development Setup
+
+This is a monorepo using pnpm workspaces and Turborepo. To get started:
+
+1. Install dependencies:
+   ```sh
+   pnpm install
+   ```
+
+2. Build all packages:
+   ```sh
+   pnpm build
+   ```
 
-<img width="707" alt="Screenshot 2024-04-13 at 07 16 21" src="https://github.com/different-ai/file-organizer-2000/assets/11430621/ca2222c9-cb8d-4d15-8459-2da4c9662f24">
+3. Development commands:
+   - Start web development server: `cd packages/web && pnpm dev`
+   - Build plugin: `cd packages/plugin && pnpm build`
+   - Build audio server: `cd packages/audio-server && pnpm build`
 
-## Testing Different models
+The project consists of the following packages:
+- `packages/plugin`: The Obsidian plugin
+- `packages/web`: The web application
+- `packages/audio-server`: Audio transcription server
+- `packages/shared`: Shared utilities and types
 
 For the tech-savvies who would like to play around with different models, there is a promptfoo.yaml file in the project including examples with local LLMs.
 See link for more info: https://promptfoo.dev/docs/configuration/guide/

package.json

@@ -1,69 +1,15 @@
 {
-  "name": "fileorganizer2000",
-  "version": "1.131.1",
-  "description": "AI File Organizer 2000",
-  "main": "main.js",
+  "name": "file-organizer-2000-monorepo",
+  "private": true,
   "scripts": {
-    "dev": "node esbuild.config.mjs",
-    "build": "node esbuild.config.mjs production",
-    "version": "node version-bump.mjs && git add manifest.json versions.json",
-    "test": "ts-node ./standalone/index.test.ts",
-    "release": "node release.js"
+    "build": "turbo run build",
+    "dev": "turbo run dev",
+    "lint": "turbo run lint",
+    "test": "turbo run test"
   },
-  "author": "",
-  "license": "MIT",
   "devDependencies": {
-    "@types/node": "^16.11.6",
-    "@types/react": "^18.3.1",
-    "@types/react-dom": "^18.3.0",
-    "@typescript-eslint/eslint-plugin": "5.29.0",
-    "@typescript-eslint/parser": "5.29.0",
-    "autoprefixer": "^10.4.20",
-    "builtin-modules": "3.3.0",
-    "esbuild": "0.17.3",
-    "esbuild-postcss": "^0.0.4",
-    "obsidian": "^1.7.2",
-    "postcss": "^8.4.31",
-    "tailwindcss": "^3.4.14",
-    "tslib": "2.4.0",
+    "turbo": "latest",
     "typescript": "^5.2.2"
   },
-  "dependencies": {
-    "@ai-sdk/openai": "^0.0.40",
-    "@ai-sdk/react": "^0.0.70",
-    "@emotion/styled": "^11.13.0",
-    "@tiptap/core": "^2.5.7",
-    "@tiptap/extension-mention": "^2.5.8",
-    "@tiptap/pm": "^2.5.7",
-    "@tiptap/react": "^2.5.7",
-    "@tiptap/starter-kit": "^2.5.7",
-    "ai": "^3.4.32",
-    "axios": "^1.6.2",
-    "chokidar": "^3.6.0",
-    "clsx": "^2.1.1",
-    "compromise": "^14.14.2",
-    "form-data": "^4.0.0",
-    "framer-motion": "^11.5.4",
-    "fuse.js": "^7.0.0",
-    "jimp": "^0.22.12",
-    "lodash": "^4.17.21",
-    "lucide-react": "^0.441.0",
-    "moment": "^2.29.4",
-    "natural": "^8.0.1",
-    "node-fetch": "^3.3.2",
-    "ollama-ai-provider": "^0.15.2",
-    "openai": "^4.19.0",
-    "react": "^18.3.1",
-    "react-dom": "^18.3.1",
-    "react-markdown": "^9.0.1",
-    "tailwind-merge": "^2.5.4",
-    "tiktoken": "^1.0.15",
-    "tippy.js": "^6.3.7",
-    "tiptap-markdown": "^0.8.10",
-    "ts-node": "^10.9.2",
-    "use-debounce": "^10.0.4",
-    "youtube-transcript": "^1.2.1",
-    "zod": "^3.23.8",
-    "zustand": "^5.0.1"
-  }
+  "packageManager": "pnpm@8.15.4"
 }

packages/audio-server/dist/server.d.ts

@@ -0,0 +1 @@
+export {};

packages/audio-server/dist/server.js

@@ -0,0 +1,151 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const dotenv_1 = __importDefault(require("dotenv"));
+const express_1 = __importDefault(require("express"));
+const multer_1 = __importDefault(require("multer"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const cors_1 = __importDefault(require("cors"));
+const sdk_1 = require("@deepgram/sdk");
+const api_1 = require("@unkey/api");
+const morgan_1 = __importDefault(require("morgan"));
+const child_process_1 = require("child_process");
+const util_1 = __importDefault(require("util"));
+dotenv_1.default.config();
+const app = (0, express_1.default)();
+const storage = multer_1.default.diskStorage({
+    destination: function (req, file, cb) {
+        const uploadDir = path_1.default.join(__dirname, '..', 'uploads');
+        fs_1.default.mkdirSync(uploadDir, { recursive: true });
+        cb(null, uploadDir);
+    },
+    filename: function (req, file, cb) {
+        const fileExtension = file.originalname.split(".").pop();
+        cb(null, `${file.fieldname}-${Date.now()}.${fileExtension}`);
+    },
+});
+const upload = (0, multer_1.default)({ storage: storage });
+const deepgram = (0, sdk_1.createClient)(process.env.DEEPGRAM_API_KEY || "");
+app.use((0, morgan_1.default)("dev"));
+app.use((0, cors_1.default)({
+    origin: function (origin, callback) {
+        console.log("Request origin:", origin);
+        callback(null, true);
+    },
+    methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"],
+    allowedHeaders: ["Content-Type", "Authorization"],
+}));
+app.use((req, res, next) => {
+    console.log("Incoming request:", {
+        method: req.method,
+        path: req.path,
+        headers: req.headers,
+        origin: req.get("origin"),
+    });
+    next();
+});
+app.get("/health", (req, res) => {
+    res.status(200).json({ status: "OK" });
+});
+app.get("/", (req, res) => {
+    res.status(200).send("Audio Transcription Server");
+});
+app.post("/transcribe", upload.single("audio"), async (req, res) => {
+    var _a, _b, _c, _d, _e, _f, _g;
+    console.log("receiving file");
+    const authHeader = req.headers["authorization"];
+    const key = authHeader === null || authHeader === void 0 ? void 0 : authHeader.toString().replace("Bearer ", "");
+    console.log(key);
+    if (!key) {
+        return res.status(401).send("Unauthorized");
+    }
+    const { result, error } = await (0, api_1.verifyKey)(key);
+    console.log(result);
+    if (error) {
+        console.error(error);
+        return res.status(500).send("Internal Server Error");
+    }
+    if (!result.valid) {
+        return res.status(401).send("Unauthorized");
+    }
+    console.log("receiving file");
+    if (!req.file) {
+        return res.status(400).send("No audio file uploaded.");
+    }
+    const fileExtension = (_a = req.file.filename.split(".").pop()) === null || _a === void 0 ? void 0 : _a.toLowerCase();
+    const supportedFormats = [
+        "flac",
+        "m4a",
+        "mp3",
+        "mp4",
+        "mpeg",
+        "mpga",
+        "oga",
+        "ogg",
+        "wav",
+        "webm",
+    ];
+    if (!fileExtension || !supportedFormats.includes(fileExtension)) {
+        fs_1.default.unlinkSync(req.file.path);
+        return res
+            .status(400)
+            .json({
+            error: `Unsupported file format: ${fileExtension}. Supported formats: ${supportedFormats.join(", ")}`,
+        });
+    }
+    console.log("file extension", fileExtension);
+    try {
+        const chunkDuration = 10 * 60;
+        const audioInfo = await getAudioDuration(req.file.path);
+        const totalDuration = audioInfo.duration;
+        const chunks = Math.ceil(totalDuration / chunkDuration);
+        res.setHeader("Content-Type", "text/plain");
+        res.setHeader("Transfer-Encoding", "chunked");
+        for (let i = 0; i < chunks; i++) {
+            const start = i * chunkDuration;
+            const end = Math.min((i + 1) * chunkDuration, totalDuration);
+            const chunkPath = `${req.file.path}_chunk_${i}.${fileExtension}`;
+            await splitAudio(req.file.path, chunkPath, start, end);
+            console.log("in loop");
+            const { result, error } = await deepgram.listen.prerecorded.transcribeFile(fs_1.default.readFileSync(chunkPath), {
+                model: "nova-2",
+                smart_format: true,
+                detect_language: true,
+            });
+            if (error) {
+                console.error("Deepgram transcription error:", error);
+                throw new Error("Deepgram transcription failed");
+            }
+            console.log((_d = (_c = (_b = result.results) === null || _b === void 0 ? void 0 : _b.channels[0]) === null || _c === void 0 ? void 0 : _c.alternatives[0]) === null || _d === void 0 ? void 0 : _d.transcript);
+            res.write(((_g = (_f = (_e = result.results) === null || _e === void 0 ? void 0 : _e.channels[0]) === null || _f === void 0 ? void 0 : _f.alternatives[0]) === null || _g === void 0 ? void 0 : _g.transcript) + " ");
+            fs_1.default.unlinkSync(chunkPath);
+        }
+        fs_1.default.unlinkSync(req.file.path);
+        res.end();
+    }
+    catch (error) {
+        console.error("Error transcribing audio:", error);
+        fs_1.default.unlinkSync(req.file.path);
+        res
+            .status(500)
+            .json({
+            error: "Error transcribing audio",
+            details: error.message,
+        });
+    }
+});
+const PORT = process.env.PORT || 3001;
+app.listen(PORT, () => console.log(`Server running on port ${PORT}`));
+const execPromise = util_1.default.promisify(child_process_1.exec);
+async function getAudioDuration(filePath) {
+    const command = `ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "${filePath}"`;
+    const { stdout } = await execPromise(command);
+    return { duration: parseFloat(stdout) };
+}
+async function splitAudio(inputPath, outputPath, start, end) {
+    const command = `ffmpeg -i "${inputPath}" -ss ${start} -to ${end} -c copy "${outputPath}"`;
+    await execPromise(command);
+}

audio-transcription-server/package.json → packages/audio-server/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "audio-transcription-server",
+  "name": "@file-organizer/audio-server",
   "version": "1.0.0",
   "description": "",
   "main": "dist/server.js",

packages/audio-server/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "target": "es2018",
+    "module": "commonjs",
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "composite": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules"],
+  "references": [
+    { "path": "../shared" }
+  ]
+}

packages/plugin/.turbo/turbo-test.log

@@ -0,0 +1,30 @@
+
+
+> @file-organizer/plugin@1.131.1 test /home/ubuntu/repos/file-organizer-2000/packages/plugin
+> ts-node ./standalone/index.test.ts
+
+node:internal/modules/cjs/loader:1252
+  throw err;
+  ^
+
+Error: Cannot find module './index.test.ts'
+Require stack:
+- /home/ubuntu/repos/file-organizer-2000/packages/plugin/standalone/imaginaryUncacheableRequireResolveScript
+    at Function._resolveFilename (node:internal/modules/cjs/loader:1249:15)
+    at Function.resolve (node:internal/modules/helpers:151:19)
+    at requireResolveNonCached (/home/ubuntu/repos/file-organizer-2000/node_modules/.pnpm/ts-node@10.9.2_@types+node@20.8.5_typescript@5.2.2/node_modules/ts-node/dist/bin.js:549:16)
+    at getProjectSearchDir (/home/ubuntu/repos/file-organizer-2000/node_modules/.pnpm/ts-node@10.9.2_@types+node@20.8.5_typescript@5.2.2/node_modules/ts-node/dist/bin.js:519:40)
+    at phase3 (/home/ubuntu/repos/file-organizer-2000/node_modules/.pnpm/ts-node@10.9.2_@types+node@20.8.5_typescript@5.2.2/node_modules/ts-node/dist/bin.js:267:27)
+    at bootstrap (/home/ubuntu/repos/file-organizer-2000/node_modules/.pnpm/ts-node@10.9.2_@types+node@20.8.5_typescript@5.2.2/node_modules/ts-node/dist/bin.js:47:30)
+    at main (/home/ubuntu/repos/file-organizer-2000/node_modules/.pnpm/ts-node@10.9.2_@types+node@20.8.5_typescript@5.2.2/node_modules/ts-node/dist/bin.js:33:12)
+    at Object.<anonymous> (/home/ubuntu/repos/file-organizer-2000/node_modules/.pnpm/ts-node@10.9.2_@types+node@20.8.5_typescript@5.2.2/node_modules/ts-node/dist/bin.js:579:5)
+    at Module._compile (node:internal/modules/cjs/loader:1546:14)
+    at Object..js (node:internal/modules/cjs/loader:1689:10) {
+  code: 'MODULE_NOT_FOUND',
+  requireStack: [
+    '/home/ubuntu/repos/file-organizer-2000/packages/plugin/standalone/imaginaryUncacheableRequireResolveScript'
+  ]
+}
+
+Node.js v22.11.0
+ ELIFECYCLE  Test failed. See above for more details.