feat(db-mongodb): migrate relationships v2-v3 (#9182)

- Adds predefined migration `@payloadcms/db-mongodb/relationships-v2-v3`
that converts all string relationship values to ObjectIDs.
- Fixes / refactors `versions-v1-v2` migration, ensures the transaction is
used
- Adds tests for Mongoose predefined migrations

packages/db-mongodb/package.json

@@ -23,6 +23,11 @@
       "import": "./src/index.ts",
       "types": "./src/index.ts",
       "default": "./src/index.ts"
+    },
+    "./migration-utils": {
+      "import": "./src/exports/migration-utils.ts",
+      "types": "./src/exports/migration-utils.ts",
+      "default": "./src/exports/migration-utils.ts"
     }
   },
   "main": "./src/index.ts",
@@ -65,6 +70,11 @@
         "import": "./dist/index.js",
         "types": "./dist/index.d.ts",
         "default": "./dist/index.js"
+      },
+      "./migration-utils": {
+        "import": "./dist/exports/migration-utils.js",
+        "types": "./dist/exports/migration-utils.d.ts",
+        "default": "./dist/exports/migration-utils.js"
       }
     },
     "main": "./dist/index.js",

packages/db-mongodb/src/exports/migration-utils.ts

@@ -0,0 +1,2 @@
+export { migrateRelationshipsV2_V3 } from '../predefinedMigrations/migrateRelationshipsV2_V3.js'
+export { migrateVersionsV1_V2 } from '../predefinedMigrations/migrateVersionsV1_V2.js'

packages/db-mongodb/src/predefinedMigrations/migrateRelationshipsV2_V3.ts

@@ -0,0 +1,169 @@
+import type { ClientSession, Model } from 'mongoose'
+import type { Field, PayloadRequest, SanitizedConfig } from 'payload'
+
+import { buildVersionCollectionFields, buildVersionGlobalFields } from 'payload'
+
+import type { MongooseAdapter } from '../index.js'
+
+import { sanitizeRelationshipIDs } from '../utilities/sanitizeRelationshipIDs.js'
+import { withSession } from '../withSession.js'
+
+const migrateModelWithBatching = async ({
+  batchSize,
+  config,
+  fields,
+  Model,
+  session,
+}: {
+  batchSize: number
+  config: SanitizedConfig
+  fields: Field[]
+  Model: Model<any>
+  session: ClientSession
+}): Promise<void> => {
+  let hasNext = true
+  let skip = 0
+
+  while (hasNext) {
+    const docs = await Model.find({}, {}, { lean: true, limit: batchSize + 1, session, skip })
+    hasNext = docs.length > batchSize
+
+    if (hasNext) {
+      docs.pop()
+    }
+
+    for (const doc of docs) {
+      sanitizeRelationshipIDs({ config, data: doc, fields })
+    }
+
+    await Model.bulkWrite(
+      docs.map((doc) => ({
+        updateOne: {
+          filter: { _id: doc._id },
+          update: doc,
+        },
+      })),
+      { session },
+    )
+
+    skip += batchSize
+  }
+}
+
+const hasRelationshipOrUploadField = ({ fields }: { fields: Field[] }): boolean => {
+  for (const field of fields) {
+    if (field.type === 'relationship' || field.type === 'upload') {
+      return true
+    }
+
+    if ('fields' in field) {
+      if (hasRelationshipOrUploadField({ fields: field.fields })) {
+        return true
+      }
+    }
+
+    if ('blocks' in field) {
+      for (const block of field.blocks) {
+        if (hasRelationshipOrUploadField({ fields: block.fields })) {
+          return true
+        }
+      }
+    }
+
+    if ('tabs' in field) {
+      for (const tab of field.tabs) {
+        if (hasRelationshipOrUploadField({ fields: tab.fields })) {
+          return true
+        }
+      }
+    }
+  }
+
+  return false
+}
+
+export async function migrateRelationshipsV2_V3({
+  batchSize,
+  req,
+}: {
+  batchSize: number
+  req: PayloadRequest
+}): Promise<void> {
+  const { payload } = req
+  const db = payload.db as MongooseAdapter
+  const config = payload.config
+
+  const { session } = await withSession(db, req)
+
+  for (const collection of payload.config.collections.filter(hasRelationshipOrUploadField)) {
+    payload.logger.info(`Migrating collection "${collection.slug}"`)
+
+    await migrateModelWithBatching({
+      batchSize,
+      config,
+      fields: collection.fields,
+      Model: db.collections[collection.slug],
+      session,
+    })
+
+    payload.logger.info(`Migrated collection "${collection.slug}"`)
+
+    if (collection.versions) {
+      payload.logger.info(`Migrating collection versions "${collection.slug}"`)
+
+      await migrateModelWithBatching({
+        batchSize,
+        config,
+        fields: buildVersionCollectionFields(config, collection),
+        Model: db.versions[collection.slug],
+        session,
+      })
+
+      payload.logger.info(`Migrated collection versions "${collection.slug}"`)
+    }
+  }
+
+  const { globals: GlobalsModel } = db
+
+  for (const global of payload.config.globals.filter(hasRelationshipOrUploadField)) {
+    payload.logger.info(`Migrating global "${global.slug}"`)
+
+    const doc = await GlobalsModel.findOne<Record<string, unknown>>(
+      {
+        globalType: {
+          $eq: global.slug,
+        },
+      },
+      {},
+      { lean: true, session },
+    )
+
+    sanitizeRelationshipIDs({ config, data: doc, fields: global.fields })
+
+    await GlobalsModel.updateOne(
+      {
+        globalType: {
+          $eq: global.slug,
+        },
+      },
+      doc,
+      { session },
+    )
+
+    payload.logger.info(`Migrated global "${global.slug}"`)
+
+    if (global.versions) {
+      payload.logger.info(`Migrating global versions "${global.slug}"`)
+
+      await migrateModelWithBatching({
+        batchSize,
+        config,
+        fields: buildVersionGlobalFields(config, global),
+        Model: db.versions[global.slug],
+        session,
+      })
+
+      payload.logger.info(`Migrated global versions "${global.slug}"`)
+    }
+  }
+}

packages/db-mongodb/src/predefinedMigrations/migrateVersionsV1_V2.ts

@@ -0,0 +1,126 @@
+import type { ClientSession } from 'mongoose'
+import type { Payload, PayloadRequest } from 'payload'
+
+import type { MongooseAdapter } from '../index.js'
+
+import { withSession } from '../withSession.js'
+
+export async function migrateVersionsV1_V2({ req }: { req: PayloadRequest }) {
+  const { payload } = req
+
+  const { session } = await withSession(payload.db as MongooseAdapter, req)
+
+  // For each collection
+
+  for (const { slug, versions } of payload.config.collections) {
+    if (versions?.drafts) {
+      await migrateCollectionDocs({ slug, payload, session })
+
+      payload.logger.info(`Migrated the "${slug}" collection.`)
+    }
+  }
+
+  // For each global
+  for (const { slug, versions } of payload.config.globals) {
+    if (versions) {
+      const VersionsModel = payload.db.versions[slug]
+
+      await VersionsModel.findOneAndUpdate(
+        {},
+        { latest: true },
+        {
+          session,
+          sort: { updatedAt: -1 },
+        },
+      ).exec()
+
+      payload.logger.info(`Migrated the "${slug}" global.`)
+    }
+  }
+}
+
+async function migrateCollectionDocs({
+  slug,
+  docsAtATime = 100,
+  payload,
+  session,
+}: {
+  docsAtATime?: number
+  payload: Payload
+  session: ClientSession
+  slug: string
+}) {
+  const VersionsModel = payload.db.versions[slug]
+  const remainingDocs = await VersionsModel.aggregate(
+    [
+      // Sort so that newest are first
+      {
+        $sort: {
+          updatedAt: -1,
+        },
+      },
+      // Group by parent ID
+      // take the $first of each
+      {
+        $group: {
+          _id: '$parent',
+          _versionID: { $first: '$_id' },
+          createdAt: { $first: '$createdAt' },
+          latest: { $first: '$latest' },
+          updatedAt: { $first: '$updatedAt' },
+          version: { $first: '$version' },
+        },
+      },
+      {
+        $match: {
+          latest: { $eq: null },
+        },
+      },
+      {
+        $limit: docsAtATime,
+      },
+    ],
+    {
+      allowDiskUse: true,
+      session,
+    },
+  ).exec()
+
+  if (!remainingDocs || remainingDocs.length === 0) {
+    const newVersions = await VersionsModel.find(
+      {
+        latest: {
+          $eq: true,
+        },
+      },
+      undefined,
+      { session },
+    )
+
+    if (newVersions?.length) {
+      payload.logger.info(
+        `Migrated ${newVersions.length} documents in the "${slug}" versions collection.`,
+      )
+    }
+
+    return
+  }
+
+  const remainingDocIds = remainingDocs.map((doc) => doc._versionID)
+
+  await VersionsModel.updateMany(
+    {
+      _id: {
+        $in: remainingDocIds,
+      },
+    },
+    {
+      latest: true,
+    },
+    {
+      session,
+    },
+  )
+
+  await migrateCollectionDocs({ slug, payload, session })
+}

packages/db-mongodb/src/predefinedMigrations/relationships-v2-v3.ts

@@ -0,0 +1,9 @@
+const imports = `import { migrateRelationshipsV2_V3 } from '@payloadcms/db-mongodb/migration-utils'`
+const upSQL = `   await migrateRelationshipsV2_V3({
+        batchSize: 100,
+        req,
+        })
+`
+export { imports, upSQL }
+
+//# sourceMappingURL=versions-v2-v3.js.map