✨ feat: lobe-i18n - 添加 Markdown 文件翻译缓存功能 (#98)

packages/lobe-i18n/package.json

@@ -54,6 +54,7 @@
     "type-check": "tsc --noEmit"
   },
   "dependencies": {
+    "@deno/kv": "^0.7.0",
     "@inkjs/ui": "^1",
     "@langchain/core": "latest",
     "@langchain/openai": "latest",

packages/lobe-i18n/src/cache/index.ts

@@ -0,0 +1,38 @@
+import { Kv, openKv } from '@deno/kv';
+import crypto from 'node:crypto';
+
+export class FileHashCache {
+  kv: Promise<Kv>;
+  constructor(cacheDB = '.lobe-i18n.db') {
+    this.kv = openKv(cacheDB);
+  }
+  getHash(fileContent: string) {
+    return crypto.createHash('md5').update(fileContent).digest('hex');
+  }
+  setHash(filePath: string, key: string) {
+    return this.kv.then((kv) => {
+      return kv.set(['files', filePath], key);
+    });
+  }
+  hasHash(filePath: string, key: string) {
+    return this.kv
+      .then((kv) => {
+        return kv.get<string>(['files', filePath]);
+      })
+      .then((res) => {
+        return res.value === key;
+      });
+  }
+  remove(filePath: string) {
+    return this.kv.then((kv) => kv.delete(['files', filePath]));
+  }
+  clear() {
+    return this.kv.then(async (kv) => {
+      const iter = kv.list<string>({ prefix: ['files'] });
+      for await (const res of iter) kv.delete(res.key);
+    });
+  }
+  destory() {
+    return this.kv.then((kv) => kv.close());
+  }
+}

packages/lobe-i18n/src/cli.tsx

@@ -16,6 +16,7 @@ notifier.notify({ isGlobal: true });
 const program = new Command();
 
 interface Flags {
+  clear?: boolean;
   config?: string;
   option?: boolean;
   withMd?: boolean;
@@ -27,6 +28,7 @@ program
   .version(packageJson.version)
   .addOption(new Option('-o, --option', 'Setup lobe-i18n preferences'))
   .addOption(new Option('-c, --config <string>', 'Specify the configuration file'))
+  .addOption(new Option('--clear', 'clear cache'))
   .addOption(
     new Option('-m, --with-md', 'Run i18n translation and markdown translation simultaneously'),
   );
@@ -45,7 +47,11 @@ program.command('locale', { isDefault: true }).action(async () => {
 program.command('md').action(async () => {
   const options: Flags = program.opts();
   if (options.config) explorer.loadCustomConfig(options.config);
-  await new TranslateMarkdown().start();
+  const service = new TranslateMarkdown();
+  if (options.clear) {
+    await service.fileHashCache.clear();
+  }
+  await service.start();
 });
 
 program.parse();

packages/lobe-i18n/src/commands/TranslateMarkdown/index.tsx

@@ -4,9 +4,9 @@ import { consola } from 'consola';
 import { globSync } from 'glob';
 import matter from 'gray-matter';
 import { isString } from 'lodash-es';
-import { existsSync } from 'node:fs';
 import { relative, resolve } from 'node:path';
 
+import { FileHashCache } from '@/cache';
 import Progress from '@/components/Progress';
 import { I18n, I18nMarkdownWriteOptions } from '@/core/I18n';
 import { selectors } from '@/store';
@@ -21,6 +21,7 @@ class TranslateMarkdown {
   markdownConfig: MarkdownConfig;
   query: I18nMarkdownWriteOptions[] = [];
   i18n: I18n;
+  fileHashCache = new FileHashCache();
   constructor() {
     this.markdownConfig = selectors.getMarkdownConfigFile();
     const defaultConfig = selectors.getConfigFile();
@@ -55,7 +56,7 @@ class TranslateMarkdown {
 
     if (!files || files.length === 0) alert.error('No markdown entry was found.', true);
 
-    this.genFilesQuery(files);
+    await this.genFilesQuery(files);
 
     if (this.query.length > 0) {
       await this.runQuery();
@@ -64,7 +65,6 @@ class TranslateMarkdown {
     }
     consola.success('All i18n tasks have been completed！');
   }
-
   async runQuery() {
     consola.info(
       `Current model setting: ${chalk.cyan(this.config.modelName)} (temperature: ${chalk.cyan(
@@ -101,6 +101,7 @@ class TranslateMarkdown {
         }
 
         writeMarkdown(item.filename, mdResut);
+        await this.fileHashCache.setHash(item.filePath, item.hash!);
         totalTokenUsage += data.tokenUsage;
         consola.success(chalk.yellow(outputPath), chalk.gray(`[Token usage: ${data.tokenUsage}]`));
       } else {
@@ -110,7 +111,7 @@ class TranslateMarkdown {
     if (totalTokenUsage > 0) consola.info('Total token usage:', chalk.cyan(totalTokenUsage));
   }
 
-  genFilesQuery(files: string[], skipLog?: boolean) {
+  async genFilesQuery(files: string[], skipLog?: boolean) {
     const config = this.markdownConfig;
     if (!skipLog)
       consola.start(
@@ -121,24 +122,31 @@ class TranslateMarkdown {
     for (const file of files) {
       try {
         const md = readMarkdown(file);
+        const hash = this.fileHashCache.getHash(md);
+        const cacheKey = await this.fileHashCache.hasHash(file, hash);
+        if (cacheKey) {
+          consola.info(`✅ Cahced: ${chalk.green(file)}`);
+          continue;
+        }
         for (const locale of config.outputLocales || []) {
           const targetExtension = this.getTargetExtension(locale, file, md);
           const targetFilename = this.getTargetFilename(file, targetExtension);
-          if (existsSync(targetFilename)) continue;
           const mode = this.getMode(file, md);
           const { data, content } = matter(md);
           consola.info(`📄 To ${locale}: ${chalk.yellow(targetFilename)}`);
           this.query.push({
+            filePath: file,
             filename: targetFilename,
             from: config.entryLocale,
+            hash,
             matter: data,
             md: this.markdownConfig.includeMatter ? md : content,
             mode,
             to: locale,
           });
         }
-      } catch {
-        alert.error(`${file} not found`, true);
+      } catch (error: unknown) {
+        alert.error(`${file} not found. ` + (error as Error).message, true);
       }
     }
   }

packages/lobe-i18n/src/core/I18n.ts

@@ -33,6 +33,8 @@ export interface I18nTranslateOptions {
 
 export interface I18nMarkdownTranslateOptions
   extends Pick<I18nTranslateOptions, 'from' | 'to' | 'onProgress'> {
+  filePath: string;
+  hash: string;
   matter?: any;
   md: string;
   mode: MarkdownModeType;