Adds initial support for external attachments using the Filesystem or MinIO

app/common/DocumentSettings.ts

@@ -2,6 +2,7 @@ export interface DocumentSettings {
   locale: string;
   currency?: string;
   engine?: EngineCode;
+  idOfDefaultAttachmentStore?: string;
 }
 
 /**

app/server/generateInitialDocSql.ts

@@ -1,4 +1,5 @@
 import { ActiveDoc } from 'app/server/lib/ActiveDoc';
+import { AttachmentStoreProvider } from "app/server/lib/AttachmentStoreProvider";
 import { create } from 'app/server/lib/create';
 import { DocManager } from 'app/server/lib/DocManager';
 import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
@@ -32,11 +33,12 @@ export async function main(baseName: string) {
     if (await fse.pathExists(fname)) {
       await fse.remove(fname);
     }
-    const docManager = new DocManager(storageManager, pluginManager, null as any, {
-      create,
-      getAuditLogger() { return createDummyAuditLogger(); },
-      getTelemetry() { return createDummyTelemetry(); },
-    } as any);
+    const docManager = new DocManager(storageManager, pluginManager, null as any, new AttachmentStoreProvider([], ""), {
+        create,
+        getAuditLogger() { return createDummyAuditLogger(); },
+        getTelemetry() { return createDummyTelemetry(); },
+      } as any
+    );
     const activeDoc = new ActiveDoc(docManager, baseName);
     const session = makeExceptionalDocSession('nascent');
     await activeDoc.createEmptyDocWithDataEngine(session);

app/server/lib/ActiveDoc.ts

@@ -95,7 +95,6 @@ import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
 import {AssistanceContext} from 'app/common/AssistancePrompts';
 import {AuditEventProperties} from 'app/server/lib/AuditLogger';
 import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer';
-import {checksumFile} from 'app/server/lib/checksumFile';
 import {Client} from 'app/server/lib/Client';
 import {getMetaTables} from 'app/server/lib/DocApi';
 import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager';
@@ -137,6 +136,8 @@ import tmp from 'tmp';
 import {ActionHistory} from './ActionHistory';
 import {ActionHistoryImpl} from './ActionHistoryImpl';
 import {ActiveDocImport, FileImportOptions} from './ActiveDocImport';
+import {AttachmentFileManager} from "./AttachmentFileManager";
+import {IAttachmentStoreProvider} from "./AttachmentStoreProvider";
 import {DocClients} from './DocClients';
 import {DocPluginManager} from './DocPluginManager';
 import {DocSession, makeExceptionalDocSession, OptDocSession} from './DocSession';
@@ -154,6 +155,7 @@ import remove = require('lodash/remove');
 import sum = require('lodash/sum');
 import without = require('lodash/without');
 import zipObject = require('lodash/zipObject');
+import { readFile } from "fs-extra";
 
 bluebird.promisifyAll(tmp);
 
@@ -268,6 +270,7 @@ export class ActiveDoc extends EventEmitter {
   private _onlyAllowMetaDataActionsOnDb: boolean = false;
   // Cache of which columns are attachment columns.
   private _attachmentColumns?: AttachmentColumns;
+  private _attachmentFileManager: AttachmentFileManager;
 
   // Client watching for 'product changed' event published by Billing to update usage
   private _redisSubscriber?: RedisClient;
@@ -283,7 +286,12 @@ export class ActiveDoc extends EventEmitter {
   private _doShutdown?: Promise<void>;
   private _intervals: Interval[] = [];
 
-  constructor(docManager: DocManager, docName: string, private _options?: ICreateActiveDocOptions) {
+  constructor(
+    docManager: DocManager,
+    docName: string,
+    externalAttachmentStoreProvider?: IAttachmentStoreProvider,
+    private _options?: ICreateActiveDocOptions,
+  ) {
     super();
     const {forkId, snapshotId} = parseUrlId(docName);
     this._isSnapshot = Boolean(snapshotId);
@@ -378,6 +386,14 @@ export class ActiveDoc extends EventEmitter {
       loadTable: this._rawPyCall.bind(this, 'load_table'),
     });
 
+    // This will throw errors if _doc or externalAttachmentStoreProvider aren't provided, and ActiveDoc tries to use
+    // an external attachment store.
+    this._attachmentFileManager = new AttachmentFileManager(
+      this.docStorage,
+      externalAttachmentStoreProvider,
+      this._doc
+    );
+
     // Our DataEngine is a separate sandboxed process (one sandbox per open document,
     // corresponding to one process for pynbox, more for gvisor).
     // The data engine runs user-defined python code including formula calculations.
@@ -912,7 +928,7 @@ export class ActiveDoc extends EventEmitter {
         }
       }
     }
-    const data = await this.docStorage.getFileData(fileIdent);
+    const data = await this._attachmentFileManager.getFileData(fileIdent);
     if (!data) { throw new ApiError("Invalid attachment identifier", 404); }
     this._log.info(docSession, "getAttachment: %s -> %s bytes", fileIdent, data.length);
     return data;
@@ -2347,13 +2363,16 @@ export class ActiveDoc extends EventEmitter {
       dimensions.height = 0;
       dimensions.width = 0;
     }
-    const checksum = await checksumFile(fileData.absPath);
-    const fileIdent = checksum + fileData.ext;
-    const ret: boolean = await this.docStorage.findOrAttachFile(fileData.absPath, fileIdent);
-    this._log.info(docSession, "addAttachment: file %s (image %sx%s) %s", fileIdent,
-      dimensions.width, dimensions.height, ret ? "attached" : "already exists");
+    const attachmentStoreId = (await this._getDocumentSettings())?.idOfDefaultAttachmentStore;
+    const addFileResult = await this._attachmentFileManager
+      .addFile(attachmentStoreId, fileData.ext, await readFile(fileData.absPath));
+    this._log.info(
+      docSession, "addAttachment: store: '%s', file: '%s' (image %sx%s) %s",
+      attachmentStoreId ?? 'local document', addFileResult.fileIdent, dimensions.width, dimensions.height,
+      addFileResult.isNewFile ? "attached" : "already exists"
+    );
     return ['AddRecord', '_grist_Attachments', null, {
-      fileIdent,
+      fileIdent: addFileResult.fileIdent,
       fileName: fileData.origName,
       // We used to set fileType, but it's not easily available for native types. Since it's
       // also entirely unused, we just skip it until it becomes relevant.
@@ -2822,17 +2841,21 @@ export class ActiveDoc extends EventEmitter {
     return this._dataEngine;
   }
 
+  private async _getDocumentSettings(): Promise<DocumentSettings | undefined> {
+    const docInfo = await this.docStorage.get('SELECT documentSettings FROM _grist_DocInfo');
+    const docSettingsString = docInfo?.documentSettings;
+    return docSettingsString ? safeJsonParse(docSettingsString, undefined) : undefined;
+  }
+
   private async _makeEngine(): Promise<ISandbox> {
     // Figure out what kind of engine we need for this document.
     let preferredPythonVersion: '2' | '3' = process.env.PYTHON_VERSION === '3' ? '3' : '2';
 
     // Careful, migrations may not have run on this document and it may not have a
     // documentSettings column.  Failures are treated as lack of an engine preference.
-    const docInfo = await this.docStorage.get('SELECT documentSettings FROM _grist_DocInfo').catch(e => undefined);
-    const docSettingsString = docInfo?.documentSettings;
-    if (docSettingsString) {
-      const docSettings: DocumentSettings|undefined = safeJsonParse(docSettingsString, undefined);
-      const engine = docSettings?.engine;
+    const docSettings = await this._getDocumentSettings().catch(e => undefined);
+    if (docSettings) {
+      const engine = docSettings.engine;
       if (engine) {
         if (engine === 'python2') {
           preferredPythonVersion = '2';

app/server/lib/AttachmentFileManager.ts

@@ -0,0 +1,153 @@
+import { DocStorage } from "app/server/lib/DocStorage";
+import { checksumFileStream } from "app/server/lib/checksumFile";
+import { Readable } from "node:stream";
+import {
+  AttachmentStoreDocInfo,
+  DocPoolId,
+  getDocPoolIdFromDocInfo,
+  IAttachmentStore
+} from "app/server/lib/AttachmentStore";
+import { AttachmentStoreId, IAttachmentStoreProvider } from "app/server/lib/AttachmentStoreProvider";
+import log from 'app/server/lib/log';
+
+export interface IAttachmentFileManager {
+  addFile(storeId: AttachmentStoreId, fileExtension: string, fileData: Buffer): Promise<AddFileResult>;
+  getFileData(fileIdent: string): Promise<Buffer | null>;
+}
+
+export interface AddFileResult {
+  fileIdent: string;
+  isNewFile: boolean;
+}
+
+export class StoresNotConfiguredError extends Error {
+  constructor() {
+    super('Attempted to access a file store, but AttachmentFileManager was initialised without store access');
+  }
+}
+
+export class StoreNotAvailableError extends Error {
+  public readonly storeId: AttachmentStoreId;
+
+  constructor(storeId: AttachmentStoreId) {
+    super(`Store '${storeId}' is not a valid and available store`);
+    this.storeId = storeId;
+  }
+}
+
+/**
+ * Provides management of a specific document's attachments.
+ */
+export class AttachmentFileManager implements IAttachmentFileManager {
+  // _docPoolId is a critical point for security. Documents with a common pool id can access each others' attachments.
+  private readonly _docPoolId: DocPoolId | null;
+  private readonly _docName: string;
+
+  /**
+   * @param _docStorage - Storage of this manager's document.
+   * @param _storeProvider - Allows instantiating of stores. Should be provided except in test scenarios.
+   * @param _docInfo - The document this manager is for. Should be provided except in test scenarios.
+   */
+  constructor(
+    private _docStorage: DocStorage,
+    private _storeProvider: IAttachmentStoreProvider | undefined,
+    _docInfo: AttachmentStoreDocInfo | undefined,
+  ) {
+    this._docName = _docStorage.docName;
+    this._docPoolId = _docInfo ? getDocPoolIdFromDocInfo(_docInfo) : null;
+  }
+
+  public async addFile(
+    storeId: AttachmentStoreId | undefined,
+    fileExtension: string,
+    fileData: Buffer
+  ): Promise<AddFileResult> {
+    const fileIdent = await this._getFileIdentifier(fileExtension, Readable.from(fileData));
+    if (storeId === undefined) {
+      log.info(`AttachmentFileManager adding ${fileIdent} to local document storage in '${this._docName}'`);
+      return this._addFileToLocalStorage(fileIdent, fileData);
+    }
+    const store = await this._getStore(storeId);
+    if (!store) {
+      throw new StoreNotAvailableError(storeId);
+    }
+    log.info(`AttachmentFileManager adding ${fileIdent} to attachment store '${store?.id}', pool ${this._docPoolId}`);
+    return this._addFileToAttachmentStore(store, fileIdent, fileData);
+  }
+
+  public async getFileData(fileIdent: string): Promise<Buffer | null> {
+    const fileInfo = await this._docStorage.getFileInfo(fileIdent);
+    if (!fileInfo) {
+      log.info(`AttachmentFileManager file ${fileIdent} does not exist in doc '${this._docName}'`);
+      return null;
+    }
+    if (!fileInfo.storageId) {
+      log.info(`AttachmentFileManager retrieving ${fileIdent} from document storage in ${this._docName}`);
+      return fileInfo.data;
+    }
+    const store = await this._getStore(fileInfo.storageId);
+    if (!store) {
+      throw new StoreNotAvailableError(fileInfo.storageId);
+    }
+    log.info(
+      `AttachmentFileManager retrieving ${fileIdent} from attachment store '${store?.id}', pool ${this._docPoolId}`
+    );
+    return this._getFileDataFromAttachmentStore(store, fileIdent);
+  }
+
+  private async _addFileToLocalStorage(fileIdent: string, fileData: Buffer): Promise<AddFileResult> {
+    const isNewFile = await this._docStorage.findOrAttachFile(fileIdent, fileData);
+
+    return {
+      fileIdent,
+      isNewFile,
+    };
+  }
+
+  private async _getStore(storeId: AttachmentStoreId): Promise<IAttachmentStore | null> {
+    if (!this._storeProvider) {
+      throw new StoresNotConfiguredError();
+    }
+    return this._storeProvider.getStore(storeId);
+  }
+
+  private _getDocPoolId(): string {
+    if (!this._docPoolId) {
+      throw new StoresNotConfiguredError();
+    }
+    return this._docPoolId;
+  }
+
+  private async _getFileIdentifier(fileExtension: string, fileData: Readable): Promise<string> {
+    const checksum = await checksumFileStream(fileData);
+    return `${checksum}${fileExtension}`;
+  }
+
+  private async _addFileToAttachmentStore(store: IAttachmentStore, fileIdent: string, fileData: Buffer): Promise<AddFileResult> {
+    const isNewFile = await this._docStorage.findOrAttachFile(fileIdent, undefined, store.id);
+
+    // Verify the file exists in the store. This allows for a second attempt to correct a failed upload.
+    const existsInRemoteStorage = !isNewFile && await store.exists(this._getDocPoolId(), fileIdent);
+
+    if (!isNewFile && existsInRemoteStorage) {
+      return {
+        fileIdent,
+        isNewFile: false,
+      };
+    }
+
+    // Possible issue if this upload fails - we have the file tracked in the document, but not available in the store.
+    // TODO - Decide if we keep an entry in SQLite after an upload error or not. Probably not?
+    await store.upload(this._getDocPoolId(), fileIdent, fileData);
+
+    // TODO - Confirm in doc storage that it's successfully uploaded? Need to decide how to handle a failed upload.
+    return {
+      fileIdent,
+      isNewFile,
+    };
+  }
+
+  private async _getFileDataFromAttachmentStore(store: IAttachmentStore, fileIdent: string): Promise<Buffer> {
+    return await store.download(this._getDocPoolId(), fileIdent);
+  }
+}