Adds initial support for external attachments using the Filesystem or MinIO

app/common/DocumentSettings.ts

@@ -2,6 +2,7 @@ export interface DocumentSettings {
   locale: string;
   currency?: string;
   engine?: EngineCode;
+  attachmentStoreId?: string;
 }
 
 /**

app/server/generateInitialDocSql.ts

@@ -1,4 +1,5 @@
 import { ActiveDoc } from 'app/server/lib/ActiveDoc';
+import { AttachmentStoreProvider } from "app/server/lib/AttachmentStoreProvider";
 import { create } from 'app/server/lib/create';
 import { DocManager } from 'app/server/lib/DocManager';
 import { makeExceptionalDocSession } from 'app/server/lib/DocSession';
@@ -33,7 +34,7 @@ export async function main(baseName: string) {
     if (await fse.pathExists(fname)) {
       await fse.remove(fname);
     }
-    const docManager = new DocManager(storageManager, pluginManager, null as any, {
+    const docManager = new DocManager(storageManager, pluginManager, null as any, new AttachmentStoreProvider([], ""), {
       create,
       getAuditLogger() { return createNullAuditLogger(); },
       getTelemetry() { return createDummyTelemetry(); },

app/server/lib/ActiveDoc.ts

@@ -94,7 +94,6 @@ import {AssistanceSchemaPromptV1Context} from 'app/server/lib/Assistance';
 import {AssistanceContext} from 'app/common/AssistancePrompts';
 import {AuditEventAction} from 'app/server/lib/AuditEvent';
 import {Authorizer, RequestWithLogin} from 'app/server/lib/Authorizer';
-import {checksumFile} from 'app/server/lib/checksumFile';
 import {Client} from 'app/server/lib/Client';
 import {getMetaTables} from 'app/server/lib/DocApi';
 import {DEFAULT_CACHE_TTL, DocManager} from 'app/server/lib/DocManager';
@@ -127,6 +126,7 @@ import assert from 'assert';
 import {Mutex} from 'async-mutex';
 import * as bluebird from 'bluebird';
 import {EventEmitter} from 'events';
+import {readFile} from 'fs-extra';
 import {IMessage, MsgType} from 'grain-rpc';
 import imageSize from 'image-size';
 import * as moment from 'moment-timezone';
@@ -137,6 +137,8 @@ import tmp from 'tmp';
 import {ActionHistory} from './ActionHistory';
 import {ActionHistoryImpl} from './ActionHistoryImpl';
 import {ActiveDocImport, FileImportOptions} from './ActiveDocImport';
+import {AttachmentFileManager} from "./AttachmentFileManager";
+import {IAttachmentStoreProvider} from "./AttachmentStoreProvider";
 import {DocClients} from './DocClients';
 import {DocPluginManager} from './DocPluginManager';
 import {DocSession, makeExceptionalDocSession, OptDocSession} from './DocSession';
@@ -265,6 +267,7 @@ export class ActiveDoc extends EventEmitter {
   private _onlyAllowMetaDataActionsOnDb: boolean = false;
   // Cache of which columns are attachment columns.
   private _attachmentColumns?: AttachmentColumns;
+  private _attachmentFileManager: AttachmentFileManager;
 
   // Client watching for 'product changed' event published by Billing to update usage
   private _redisSubscriber?: RedisClient;
@@ -283,6 +286,7 @@ export class ActiveDoc extends EventEmitter {
   constructor(
     private readonly _docManager: DocManager,
     private _docName: string,
+    externalAttachmentStoreProvider?: IAttachmentStoreProvider,
     private _options?: ICreateActiveDocOptions
   ) {
     super();
@@ -388,6 +392,14 @@ export class ActiveDoc extends EventEmitter {
       loadTable: this._rawPyCall.bind(this, 'load_table'),
     });
 
+    // This will throw errors if _options?.doc or externalAttachmentStoreProvider aren't provided,
+    // and ActiveDoc tries to use an external attachment store.
+    this._attachmentFileManager = new AttachmentFileManager(
+      this.docStorage,
+      externalAttachmentStoreProvider,
+      _options?.doc,
+    );
+
     // Our DataEngine is a separate sandboxed process (one sandbox per open document,
     // corresponding to one process for pynbox, more for gvisor).
     // The data engine runs user-defined python code including formula calculations.
@@ -636,6 +648,7 @@ export class ActiveDoc extends EventEmitter {
     useExisting?: boolean,       // If set, document can be created as an overlay on
                                  // an existing sqlite file.
   }): Promise<ActiveDoc> {
+    console.log(`OPENOPENOPENOPENOPENOPENOPENOPEN ${this.docName} ANYTHING ELSE`);
     const startTime = Date.now();
     this._log.debug(docSession, "loadDoc");
     try {
@@ -925,7 +938,7 @@ export class ActiveDoc extends EventEmitter {
         }
       }
     }
-    const data = await this.docStorage.getFileData(fileIdent);
+    const data = await this._attachmentFileManager.getFileData(fileIdent);
     if (!data) { throw new ApiError("Invalid attachment identifier", 404); }
     this._log.info(docSession, "getAttachment: %s -> %s bytes", fileIdent, data.length);
     return data;
@@ -2344,13 +2357,16 @@ export class ActiveDoc extends EventEmitter {
       dimensions.height = 0;
       dimensions.width = 0;
     }
-    const checksum = await checksumFile(fileData.absPath);
-    const fileIdent = checksum + fileData.ext;
-    const ret: boolean = await this.docStorage.findOrAttachFile(fileData.absPath, fileIdent);
-    this._log.info(docSession, "addAttachment: file %s (image %sx%s) %s", fileIdent,
-      dimensions.width, dimensions.height, ret ? "attached" : "already exists");
+    const attachmentStoreId = (await this._getDocumentSettings()).attachmentStoreId;
+    const addFileResult = await this._attachmentFileManager
+      .addFile(attachmentStoreId, fileData.ext, await readFile(fileData.absPath));
+    this._log.info(
+      docSession, "addAttachment: store: '%s', file: '%s' (image %sx%s) %s",
+      attachmentStoreId ?? 'local document', addFileResult.fileIdent, dimensions.width, dimensions.height,
+      addFileResult.isNewFile ? "attached" : "already exists"
+    );
     return ['AddRecord', '_grist_Attachments', null, {
-      fileIdent,
+      fileIdent: addFileResult.fileIdent,
       fileName: fileData.origName,
       // We used to set fileType, but it's not easily available for native types. Since it's
       // also entirely unused, we just skip it until it becomes relevant.
@@ -2822,17 +2838,25 @@ export class ActiveDoc extends EventEmitter {
     return this._dataEngine;
   }
 
+  private async _getDocumentSettings(): Promise<DocumentSettings> {
+    const docInfo = await this.docStorage.get('SELECT documentSettings FROM _grist_DocInfo');
+    const docSettingsString = docInfo?.documentSettings;
+    const docSettings = docSettingsString ? safeJsonParse(docSettingsString, undefined) : undefined;
+    if (!docSettings) {
+      throw new Error("No document settings found");
+    }
+    return docSettings;
+  }
+
   private async _makeEngine(): Promise<ISandbox> {
     // Figure out what kind of engine we need for this document.
     let preferredPythonVersion: '2' | '3' = process.env.PYTHON_VERSION === '2' ? '2' : '3';
 
     // Careful, migrations may not have run on this document and it may not have a
     // documentSettings column.  Failures are treated as lack of an engine preference.
-    const docInfo = await this.docStorage.get('SELECT documentSettings FROM _grist_DocInfo').catch(e => undefined);
-    const docSettingsString = docInfo?.documentSettings;
-    if (docSettingsString) {
-      const docSettings: DocumentSettings|undefined = safeJsonParse(docSettingsString, undefined);
-      const engine = docSettings?.engine;
+    const docSettings = await this._getDocumentSettings().catch(e => undefined);
+    if (docSettings) {
+      const engine = docSettings.engine;
       if (engine) {
         if (engine === 'python2') {
           preferredPythonVersion = '2';

app/server/lib/AttachmentFileManager.ts

@@ -0,0 +1,217 @@
+import {
+  AttachmentStoreDocInfo,
+  DocPoolId,
+  getDocPoolIdFromDocInfo,
+  IAttachmentStore
+} from "app/server/lib/AttachmentStore";
+import { AttachmentStoreId, IAttachmentStoreProvider } from "app/server/lib/AttachmentStoreProvider";
+import { checksumFileStream } from "app/server/lib/checksumFile";
+import { DocStorage } from "app/server/lib/DocStorage";
+import log from "app/server/lib/log";
+import { LogMethods } from "app/server/lib/LogMethods";
+import { MemoryWritableStream } from "app/server/utils/MemoryWritableStream";
+import { Readable } from "node:stream";
+
+export interface IAttachmentFileManager {
+  addFile(storeId: AttachmentStoreId, fileExtension: string, fileData: Buffer): Promise<AddFileResult>;
+  getFileData(fileIdent: string): Promise<Buffer | null>;
+}
+
+export interface AddFileResult {
+  fileIdent: string;
+  isNewFile: boolean;
+}
+
+export class StoresNotConfiguredError extends Error {
+  constructor() {
+    super('Attempted to access a file store, but AttachmentFileManager was initialized without store access');
+  }
+}
+
+export class StoreNotAvailableError extends Error {
+  public readonly storeId: AttachmentStoreId;
+
+  constructor(storeId: AttachmentStoreId) {
+    super(`Store '${storeId}' is not a valid and available store`);
+    this.storeId = storeId;
+  }
+}
+
+export class MissingAttachmentError extends Error {
+  public readonly fileIdent: string;
+
+  constructor(fileIdent: string) {
+    super(`Attachment file '${fileIdent}' could not be found in this document`);
+    this.fileIdent = fileIdent;
+  }
+}
+
+export class AttachmentRetrievalError extends Error {
+  public readonly storeId: AttachmentStoreId;
+  public readonly fileId: string;
+
+  constructor(storeId: AttachmentStoreId, fileId: string, cause?: any) {
+    const causeError = cause instanceof Error ? cause : undefined;
+    const causeDescriptor = causeError ? `: ${cause.message}` : '';
+    super(`Unable to retrieve '${fileId}' from '${storeId}'${causeDescriptor}`);
+    this.storeId = storeId;
+    this.fileId = fileId;
+    this.cause = causeError;
+  }
+}
+
+
+interface AttachmentFileManagerLogInfo {
+  fileIdent?: string;
+  storeId?: string | null;
+}
+
+/**
+ * Instantiated on a per-document basis to provide a document with access to its attachments.
+ * Handles attachment uploading / fetching, as well as trying to ensure consistency with the local document database,
+ * which tracks attachments and where they're stored.
+ *
+ * This class should prevent the document code from having to worry about accessing the underlying stores.
+ */
+export class AttachmentFileManager implements IAttachmentFileManager {
+  // _docPoolId is a critical point for security. Documents with a common pool id can access each others' attachments.
+  private readonly _docPoolId: DocPoolId | null;
+  private readonly _docName: string;
+  private _log = new LogMethods(
+    "AttachmentFileManager ",
+    (logInfo: AttachmentFileManagerLogInfo) => this._getLogMeta(logInfo)
+  );
+
+  /**
+   * @param _docStorage - Storage of this manager's document.
+   * @param _storeProvider - Allows instantiating of stores. Should be provided except in test scenarios.
+   * @param _docInfo - The document this manager is for. Should be provided except in test scenarios.
+   */
+  constructor(
+    private _docStorage: DocStorage,
+    private _storeProvider: IAttachmentStoreProvider | undefined,
+    _docInfo: AttachmentStoreDocInfo | undefined,
+  ) {
+    this._docName = _docStorage.docName;
+    this._docPoolId = _docInfo ? getDocPoolIdFromDocInfo(_docInfo) : null;
+  }
+
+  public async addFile(
+    storeId: AttachmentStoreId | undefined,
+    fileExtension: string,
+    fileData: Buffer
+  ): Promise<AddFileResult> {
+    const fileIdent = await this._getFileIdentifier(fileExtension, Readable.from(fileData));
+    return this._addFile(storeId, fileIdent, fileData);
+  }
+
+  public async _addFile(
+    storeId: AttachmentStoreId | undefined,
+    fileIdent: string,
+    fileData: Buffer
+  ): Promise<AddFileResult> {
+    this._log.info({ fileIdent, storeId }, `adding file to ${storeId ? "external" : "document"} storage`);
+    if (storeId === undefined) {
+      return this._addFileToLocalStorage(fileIdent, fileData);
+    }
+    const store = await this._getStore(storeId);
+    if (!store) {
+      this._log.info({ fileIdent, storeId }, "tried to fetch attachment from an unavailable store");
+      throw new StoreNotAvailableError(storeId);
+    }
+    return this._addFileToAttachmentStore(store, fileIdent, fileData);
+  }
+
+  public async getFileData(fileIdent: string): Promise<Buffer> {
+    const fileInfo = await this._docStorage.getFileInfo(fileIdent);
+    if (!fileInfo) {
+      this._log.error({ fileIdent }, "cannot find file metadata in document");
+      throw new MissingAttachmentError(fileIdent);
+    }
+    this._log.debug(
+      { fileIdent, storeId: fileInfo.storageId },
+      `fetching attachment from ${fileInfo.storageId ? "external" : "document "} storage`
+    );
+    if (!fileInfo.storageId) {
+      return fileInfo.data;
+    }
+    const store = await this._getStore(fileInfo.storageId);
+    if (!store) {
+      this._log.warn({ fileIdent, storeId: fileInfo.storageId }, `unable to retrieve file, store is unavailable`);
+      throw new StoreNotAvailableError(fileInfo.storageId);
+    }
+    return this._getFileDataFromAttachmentStore(store, fileIdent);
+  }
+
+  private async _addFileToLocalStorage(fileIdent: string, fileData: Buffer): Promise<AddFileResult> {
+    const isNewFile = await this._docStorage.findOrAttachFile(fileIdent, fileData);
+
+    return {
+      fileIdent,
+      isNewFile,
+    };
+  }
+
+  private async _getStore(storeId: AttachmentStoreId): Promise<IAttachmentStore | null> {
+    if (!this._storeProvider) {
+      throw new StoresNotConfiguredError();
+    }
+    return this._storeProvider.getStore(storeId);
+  }
+
+  private _getDocPoolId(): string {
+    if (!this._docPoolId) {
+      throw new StoresNotConfiguredError();
+    }
+    return this._docPoolId;
+  }
+
+  private async _getFileIdentifier(fileExtension: string, fileData: Readable): Promise<string> {
+    const checksum = await checksumFileStream(fileData);
+    return `${checksum}${fileExtension}`;
+  }
+
+  private async _addFileToAttachmentStore(
+    store: IAttachmentStore, fileIdent: string, fileData: Buffer
+  ): Promise<AddFileResult> {
+    const isNewFile = await this._docStorage.findOrAttachFile(fileIdent, undefined, store.id);
+
+    // Verify the file exists in the store. This allows for a second attempt to correct a failed upload.
+    const existsInRemoteStorage = !isNewFile && await store.exists(this._getDocPoolId(), fileIdent);
+
+    if (!isNewFile && existsInRemoteStorage) {
+      return {
+        fileIdent,
+        isNewFile: false,
+      };
+    }
+
+    // Possible issue if this upload fails - we have the file tracked in the document, but not available in the store.
+    // TODO - Decide if we keep an entry in SQLite after an upload error or not. Probably not?
+    await store.upload(this._getDocPoolId(), fileIdent, Readable.from(fileData));
+
+    // TODO - Confirm in doc storage that it's successfully uploaded? Need to decide how to handle a failed upload.
+    return {
+      fileIdent,
+      isNewFile,
+    };
+  }
+
+  private async _getFileDataFromAttachmentStore(store: IAttachmentStore, fileIdent: string): Promise<Buffer> {
+    try {
+      const outputStream = new MemoryWritableStream();
+      await store.download(this._getDocPoolId(), fileIdent, outputStream);
+      return outputStream.getBuffer();
+    } catch(e) {
+      throw new AttachmentRetrievalError(store.id, fileIdent, e);
+    }
+  }
+
+  private _getLogMeta(logInfo?: AttachmentFileManagerLogInfo): log.ILogMeta {
+    return {
+      docName: this._docName,
+      docPoolId: this._docPoolId,
+      ...logInfo,
+    };
+  }
+}