feat: TS extractors are back, take 2 (#135)

* feat: TS extractors are back * feat: tinyglobby, misc fixes, deps * fix: make all tests pass on Windows
tolgee · Dec 4, 2024 · 4fa695b · 4fa695b
1 parent 8414b49
commit 4fa695b
Show file tree

Hide file tree

Showing 14 changed files with 1,870 additions and 2,317 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -34,40 +34,49 @@
     "base32-decode": "^1.0.0",
     "commander": "^12.1.0",
     "cosmiconfig": "^9.0.0",
-    "glob": "^11.0.0",
     "json5": "^2.2.3",
     "jsonschema": "^1.4.1",
-    "openapi-fetch": "^0.10.6",
+    "openapi-fetch": "0.13.1",
+    "tinyglobby": "^0.2.10",
     "unescape-js": "^1.1.4",
     "vscode-oniguruma": "^2.0.1",
     "vscode-textmate": "^9.1.0",
-    "yauzl": "^3.1.3"
+    "yauzl": "^3.2.0"
   },
   "devDependencies": {
-    "@eslint/js": "^9.8.0",
+    "@eslint/js": "^9.16.0",
     "@semantic-release/changelog": "^6.0.3",
     "@semantic-release/git": "^10.0.1",
     "@tsconfig/node18": "^18.2.4",
-    "@tsconfig/recommended": "^1.0.7",
+    "@tsconfig/recommended": "^1.0.8",
     "@types/eslint__js": "^8.42.3",
     "@types/js-yaml": "^4.0.9",
-    "@types/node": "^22.1.0",
+    "@types/node": "^22.10.1",
     "@types/yauzl": "^2.10.3",
     "cross-env": "^7.0.3",
-    "eslint": "^9.8.0",
+    "eslint": "^9.16.0",
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.1",
+    "jiti": "^2.4.1",
     "js-yaml": "^4.1.0",
-    "json-schema-to-typescript": "^15.0.0",
-    "openapi-typescript": "^7.3.0",
+    "json-schema-to-typescript": "^15.0.3",
+    "openapi-typescript": "^7.4.4",
     "premove": "^4.0.0",
-    "prettier": "^3.3.3",
-    "semantic-release": "^24.0.0",
+    "prettier": "^3.4.1",
+    "semantic-release": "^24.2.0",
     "tree-cli": "^0.6.7",
-    "tsx": "^4.17.0",
-    "typescript": "^5.5.4",
-    "typescript-eslint": "^8.0.1",
-    "vitest": "^2.0.5"
+    "tsx": "^4.19.2",
+    "typescript": "~5.6.3",
+    "typescript-eslint": "^8.17.0",
+    "vitest": "^2.1.8"
+  },
+  "peerDependencies": {
+    "jiti": ">= 2"
+  },
+  "peerDependenciesMeta": {
+    "jiti": {
+      "optional": true
+    }
   },
   "engines": {
     "node": ">= 18"

diff --git a/src/commands/push.ts b/src/commands/push.ts
@@ -3,6 +3,7 @@ import type { BaseOptions } from '../options.js';
 import { extname, join } from 'path';
 import { readdir, readFile, stat } from 'fs/promises';
 import { Command, Option } from 'commander';
+import { glob } from 'tinyglobby';
 
 import {
   loading,
@@ -16,7 +17,6 @@ import { askString } from '../utils/ask.js';
 import { mapImportFormat } from '../utils/mapImportFormat.js';
 import { TolgeeClient, handleLoadableError } from '../client/TolgeeClient.js';
 import { BodyOf } from '../client/internal/schema.utils.js';
-import { windowsCompatibleGlob } from '../utils/windowsCompatibleGlob.js';
 
 type ImportRequest = BodyOf<
   '/v2/projects/{projectId}/single-step-import',
@@ -46,7 +46,7 @@ type PushOptions = BaseOptions & {
 
 async function allInPattern(pattern: string) {
   const files: File[] = [];
-  const items = await windowsCompatibleGlob(pattern);
+  const items = await glob(pattern);
   for (const item of items) {
     if ((await stat(item)).isDirectory()) {
       files.push(...(await readDirectory(item)));

diff --git a/src/config/tolgeerc.ts b/src/config/tolgeerc.ts
@@ -37,7 +37,7 @@ function parseConfig(input: Schema, configDir: string): Schema {
 
   // convert relative paths in config to absolute
   if (rc.extractor !== undefined) {
-    rc.extractor = resolve(configDir, rc.extractor);
+    rc.extractor = resolve(configDir, rc.extractor).replace(/\\/g, '/');
     if (!existsSync(rc.extractor)) {
       throw new Error(
         `Invalid config: extractor points to a file that does not exists (${rc.extractor})`
@@ -48,21 +48,21 @@ function parseConfig(input: Schema, configDir: string): Schema {
   // convert relative paths in config to absolute
   if (rc.patterns !== undefined) {
     rc.patterns = rc.patterns.map((pattern: string) =>
-      resolve(configDir, pattern)
+      resolve(configDir, pattern).replace(/\\/g, '/')
     );
   }
 
   // convert relative paths in config to absolute
   if (rc.push?.files) {
     rc.push.files = rc.push.files.map((r) => ({
       ...r,
-      path: resolve(configDir, r.path),
+      path: resolve(configDir, r.path).replace(/\\/g, '/'),
     }));
   }
 
   // convert relative paths in config to absolute
   if (rc.pull?.path !== undefined) {
-    rc.pull.path = resolve(configDir, rc.pull.path);
+    rc.pull.path = resolve(configDir, rc.pull.path).replace(/\\/g, '/');
   }
 
   return rc;

diff --git a/src/extractor/runner.ts b/src/extractor/runner.ts
@@ -4,11 +4,11 @@ import type {
   ParserType,
   VerboseOption,
 } from './index.js';
+import { glob } from 'tinyglobby';
 import { extname } from 'path';
 
 import { callWorker } from './worker.js';
 import { exitWithError } from '../utils/logger.js';
-import { windowsCompatibleGlob } from '../utils/windowsCompatibleGlob.js';
 
 export const NullNamespace = Symbol('namespace.null');
 
@@ -23,16 +23,27 @@ function parseVerbose(v: VerboseOption[] | boolean | undefined) {
 
 export async function extractKeysFromFile(
   file: string,
-  parserType: ParserType,
+  parserType: ParserType | undefined,
   options: ExtractOptions,
-  extractor?: string
+  extractor: string | undefined
 ) {
-  return callWorker({
-    extractor: extractor,
-    parserType,
-    file: file,
-    options,
-  });
+  if (typeof extractor !== 'undefined') {
+    return callWorker({
+      extractor,
+      file,
+      options,
+    });
+  } else if (typeof parserType !== 'undefined') {
+    return callWorker({
+      parserType,
+      file,
+      options,
+    });
+  }
+
+  throw new Error(
+    'Internal error: neither the parser type nor a custom extractors have been defined! Please report this.'
+  );
 }
 
 export function findPossibleFrameworks(fileNames: string[]) {
@@ -94,17 +105,14 @@ export async function extractKeysOfFiles(opts: Opts) {
     exitWithError("Missing '--patterns' or 'config.patterns' option");
   }
 
-  const files = await windowsCompatibleGlob(opts.patterns, {
-    nodir: true,
-  });
+  const files = await glob(opts.patterns, { onlyFiles: true });
 
   if (files.length === 0) {
     exitWithError('No files were matched for extraction');
   }
 
   let parserType = opts.parser;
-
-  if (!parserType) {
+  if (!parserType && !opts.extractor) {
     parserType = detectParserType(files);
   }
 

diff --git a/src/extractor/worker.ts b/src/extractor/worker.ts
@@ -1,49 +1,52 @@
 import type {
-  ExtractOptions,
   ExtractionResult,
+  ExtractOptions,
   Extractor,
   ParserType,
 } from './index.js';
 import { fileURLToPath } from 'url';
-import { resolve, extname } from 'path';
-import { Worker, isMainThread, parentPort } from 'worker_threads';
-import { readFile } from 'fs/promises';
+import { extname, resolve } from 'path';
+import { isMainThread, parentPort, SHARE_ENV, Worker } from 'worker_threads';
+import { readFileSync } from 'fs';
 
 import internalExtractor from './extractor.js';
 import { loadModule } from '../utils/moduleLoader.js';
-import { type Deferred, createDeferred } from '../utils/deferred.js';
+import { createDeferred, type Deferred } from '../utils/deferred.js';
 
 const FILE_TIME_LIMIT = 60 * 1000; // one minute
 
-export type WorkerParams = {
-  extractor?: string;
-  file: string;
-  parserType: ParserType;
-  options: ExtractOptions;
-};
+export type WorkerParams =
+  | {
+      file: string;
+      parserType: ParserType;
+      options: ExtractOptions;
+    }
+  | {
+      extractor: string;
+      file: string;
+      options: ExtractOptions;
+    };
 
-const IS_TS_NODE = extname(import.meta.url) === '.ts';
+const IS_TSX = extname(import.meta.url) === '.ts';
 
 // --- Worker functions
 
-let loadedExtractor: string | undefined | symbol = Symbol('unloaded');
 let extractor: Extractor;
 
 async function handleJob(args: WorkerParams): Promise<ExtractionResult> {
   const file = resolve(args.file);
-  const code = await readFile(file, 'utf8');
-  if (args.extractor) {
-    if (args.extractor !== loadedExtractor) {
-      loadedExtractor = args.extractor;
+  const code = readFileSync(file, 'utf8');
+  if ('extractor' in args) {
+    if (!extractor) {
       extractor = await loadModule(args.extractor).then((mdl) => mdl.default);
     }
     return extractor(code, file, args.options);
-  } else {
-    return internalExtractor(code, file, args.parserType, args.options);
   }
+
+  return internalExtractor(code, file, args.parserType, args.options);
 }
 
-async function workerInit() {
+function workerInit() {
   parentPort!.on('message', (params) => {
     handleJob(params)
       .then((res) => parentPort!.postMessage({ data: res }))
@@ -59,15 +62,20 @@ let worker: Worker;
 const jobQueue: Array<[WorkerParams, Deferred]> = [];
 
 function createWorker() {
-  const worker = IS_TS_NODE
-    ? new Worker(
-        fileURLToPath(new URL(import.meta.url)).replace('.ts', '.js'),
-        {
-          // ts-node workaround
-          execArgv: ['--require', 'ts-node/register'],
-        }
-      )
-    : new Worker(fileURLToPath(new URL(import.meta.url)));
+  let worker: Worker;
+  if (IS_TSX) {
+    worker = new Worker(
+      `import('tsx/esm/api').then(({ register }) => { register(); import('${fileURLToPath(new URL(import.meta.url))}') })`,
+      {
+        env: SHARE_ENV,
+        eval: true,
+      }
+    );
+  } else {
+    worker = new Worker(fileURLToPath(new URL(import.meta.url)), {
+      env: SHARE_ENV,
+    });
+  }
 
   let timeout: NodeJS.Timeout;
   let currentDeferred: Deferred;

diff --git a/src/utils/moduleLoader.ts b/src/utils/moduleLoader.ts
@@ -1,41 +1,36 @@
-import { extname } from 'path';
-
-let tsService: any;
-
-async function registerTsNode() {
-  if (!tsService) {
-    // try {
-    //   const tsNode = await import('ts-node');
-    //   tsService = tsNode.register({ compilerOptions: { module: 'CommonJS' } });
-    // } catch (e: any) {
-    //   if (e.code === 'ERR_MODULE_NOT_FOUND') {
-    //     throw new Error('ts-node is required to load TypeScript files.');
-    //   }
-    //   throw e;
-    // }
-  }
-}
+import type { Jiti } from 'jiti';
+import { pathToFileURL } from 'url';
+
+let jiti: Jiti;
 
+// https://github.com/eslint/eslint/blob/6f37b0747a14dfa9a9e3bdebc5caed1f39b6b0e2/lib/config/config-loader.js#L164-L197
 async function importTypeScript(file: string) {
-  if (extname(import.meta.url) === '.ts') {
+  // @ts-ignore
+  if (!!globalThis.Bun || !!globalThis.Deno) {
+    // We're in an env that natively supports TS
     return import(file);
   }
 
-  await registerTsNode();
+  if (!jiti) {
+    const { createJiti } = await import('jiti').catch(() => {
+      throw new Error(
+        "The 'jiti' library is required for loading TypeScript extractors. Make sure to install it."
+      );
+    });
 
-  tsService.enabled(true);
-  const mdl = await import(file);
-  tsService.enabled(false);
+    jiti = createJiti(import.meta.url);
+  }
 
-  return mdl;
+  return jiti.import(file);
 }
 
 export async function loadModule(module: string) {
   if (module.endsWith('.ts')) {
     return importTypeScript(module);
   }
 
-  const mdl = await import(module);
+  const fileUrl = pathToFileURL(module);
+  const mdl = await import(fileUrl.href);
   if (mdl.default?.default) {
     return mdl.default;
   }

diff --git a/src/utils/windowsCompatibleGlob.ts b/src/utils/windowsCompatibleGlob.ts
diff --git a/test/__fixtures__/customExtractors/extract-js.js b/test/__fixtures__/customExtractors/extract-js.js
@@ -0,0 +1,14 @@
+export default function (code) {
+  // Very simple, trivial extractor for the purposes of testing
+  const keys = [];
+  const lines = code.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    for (const [str] of lines[i].matchAll(/STR_[A-Z_]+/g)) {
+      keys.push({ keyName: str, line: i + 1 });
+    }
+  }
+
+  return {
+    keys,
+  };
+}