import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'fs'; import path from 'path'; import { APIError, buildVersionCollectionFields, buildVersionGlobalFields, dynamicImport } from 'payload'; import { findConfig } from 'payload/node'; import { fieldShouldBeLocalized } from 'payload/shared'; import { getTransaction } from './getTransaction.js'; const DEFAULT_BATCH_SIZE = 100; const TEMP_FOLDER_NAME = '.payload-blocks-migration'; const writeEntitiesToTempFile = (entities, tempFolderPath, batchIndex)=>{ const filePath = path.join(tempFolderPath, `entities-batch-${batchIndex}.json`); writeFileSync(filePath, JSON.stringify(entities, null, 2), 'utf-8'); }; const acceptDrizzlePrompts = async (callPrompt, { silenceLogs = false } = {})=>{ // eslint-disable-next-line @typescript-eslint/unbound-method const write = process.stdout.write; if (silenceLogs) { process.stdout.write = ()=>true; } const promise = callPrompt(); const interval = setInterval(()=>process.stdin.emit('keypress', '\n', { name: 'return', ctrl: false }), 25); const res = await promise; if (silenceLogs) { process.stdout.write = write; } clearInterval(interval); return res; }; const entityHasBlocksField = (entity)=>{ for (const field of entity.flattenedFields){ if (field.type === 'blocks') { return true; } if ('flattenedFields' in field && entityHasBlocksField({ flattenedFields: field.flattenedFields })) { return true; } } return false; }; const collectBlocksToMigrate = ({ config, data, fields, parentAccessor, parentIsLocalized })=>{ const result = []; for (const field of fields){ switch(field.type){ case 'array': { const arrayData = data[field.name]; if (!Array.isArray(arrayData)) { continue; } if (config.localization && fieldShouldBeLocalized({ field, parentIsLocalized })) { result.push(...collectBlocksToMigrate({ config, data: arrayData, fields: config.localization.localeCodes.map((code)=>({ ...field, name: code, localized: false })), parentAccessor: [ ...parentAccessor, field.name ], parentIsLocalized: true })); continue; } for (const [index, row] of arrayData.entries()){ result.push(...collectBlocksToMigrate({ config, data: row, fields: field.flattenedFields, parentAccessor: [ ...parentAccessor, field.name, index ], parentIsLocalized })); } break; } case 'blocks': { result.push({ data: data[field.name], fieldAccessor: [ ...parentAccessor, field.name ] }); break; } case 'group': case 'tab': { if (config.localization && fieldShouldBeLocalized({ field, parentIsLocalized })) { result.push(...collectBlocksToMigrate({ config, data: data[field.name], fields: config.localization.localeCodes.map((code)=>({ ...field, name: code, localized: false })), parentAccessor: [ ...parentAccessor, field.name ], parentIsLocalized: true })); continue; } result.push(...collectBlocksToMigrate({ config, data: data[field.name], fields: field.flattenedFields, parentAccessor: [ ...parentAccessor, field.name ], parentIsLocalized })); break; } default: { continue; } } } return result; }; class BlocksToJsonMigratorImpl { adapter; sanitizeStatements; executeMethod; batchSize; tempFolderPath; constructor(adapter, sanitizeStatements, executeMethod, batchSize){ this.adapter = adapter; this.sanitizeStatements = sanitizeStatements; this.executeMethod = executeMethod; this.batchSize = batchSize || DEFAULT_BATCH_SIZE; this.tempFolderPath = path.resolve(this.adapter.migrationDir, TEMP_FOLDER_NAME); } ensureTempFolder() { if (!existsSync(this.tempFolderPath)) { mkdirSync(this.tempFolderPath, { recursive: true }); } } *fetchEntitiesFromJsonBatches() { if (!existsSync(this.tempFolderPath)) { return; } const files = readdirSync(this.tempFolderPath).filter((file)=>file.startsWith('entities-batch-') && file.endsWith('.json')).sort((a, b)=>{ const aNum = parseInt(a.match(/\d+/)?.[0] || '0', 10); const bNum = parseInt(b.match(/\d+/)?.[0] || '0', 10); return aNum - bNum; }); for (const file of files){ const filePath = path.join(this.tempFolderPath, file); const fileContent = readFileSync(filePath, 'utf-8'); const batchEntities = JSON.parse(fileContent); yield batchEntities; } } async migrateEntities(entities, req) { this.adapter.blocksAsJSON = true; this.resetSchema(); await this.adapter.init(); await this.adapter.connect(); await this.syncTransactionDrizzleInstance(req); const totalEntities = entities.length; let processed = 0; for (const entity of entities){ switch(entity.type){ case 'collection': { await this.adapter.updateOne({ collection: entity.slug, data: entity.originalData, joins: false, req, where: { id: { equals: entity.id } } }); break; } case 'collectionVersion': { await this.adapter.updateVersion({ id: entity.id, collection: entity.slug, req, versionData: entity.originalData }); break; } case 'global': { await this.adapter.updateGlobal({ slug: entity.slug, data: entity.originalData, req }); break; } case 'globalVersion': { await this.adapter.updateGlobalVersion({ id: entity.id, global: entity.slug, req, versionData: entity.originalData }); break; } } processed++; if (processed % this.batchSize === 0 || processed === totalEntities) { this.adapter.payload.logger.info(`Migrated ${processed}/${totalEntities} entities (${Math.round(processed / totalEntities * 100)}%)`); } } } resetSchema() { this.adapter.schema = {}; this.adapter.tables = {}; this.adapter.indexes = new Set(); this.adapter.foreignKeys = new Set(); this.adapter.relations = {}; this.adapter.rawTables = {}; this.adapter.rawRelations = {}; this.adapter.tableNameMap = new Map(); this.adapter.enums = {}; } async syncTransactionDrizzleInstance(req) { const tsx = await getTransaction(this.adapter, req); tsx._ = this.adapter.drizzle._; tsx.schema = this.adapter.drizzle._; tsx.session.schema = this.adapter.drizzle.session.schema; } cleanupTempFolder() { rmSync(this.tempFolderPath, { force: true, recursive: true }); this.adapter.payload.logger.info(`Cleaned up temp folder at ${this.tempFolderPath}`); } async collectAndSaveEntitiesToBatches(req, options) { const batchSize = options?.batchSize ?? this.batchSize; this.cleanupTempFolder(); this.ensureTempFolder(); this.adapter.blocksAsJSON = false; this.resetSchema(); await this.adapter.init(); await this.adapter.connect(); // Count total entities to migrate let totalExpected = 0; for (const collection of this.adapter.payload.config.collections.filter(entityHasBlocksField)){ const { totalDocs } = await this.adapter.count({ collection: collection.slug }); totalExpected += totalDocs; if (collection.versions) { const { totalDocs: totalVersions } = await this.adapter.countVersions({ collection: collection.slug }); totalExpected += totalVersions; } } for (const globalConfig of this.adapter.payload.config.globals.filter(entityHasBlocksField)){ totalExpected += 1; // Global itself if (globalConfig.versions) { const { totalDocs: totalGlobalVersions } = await this.adapter.countGlobalVersions({ global: globalConfig.slug }); totalExpected += totalGlobalVersions; } } this.adapter.payload.logger.info(`Found ${totalExpected} total entities to collect and save to batches`); let batchIndex = 0; let currentBatch = []; let totalCollected = 0; const flushBatch = ()=>{ if (currentBatch.length > 0) { writeEntitiesToTempFile(currentBatch, this.tempFolderPath, batchIndex); const percentage = totalExpected > 0 ? Math.round(totalCollected / totalExpected * 100) : 0; this.adapter.payload.logger.info(`Saved batch ${batchIndex} with ${currentBatch.length} entities (${totalCollected}/${totalExpected} - ${percentage}%)`); batchIndex++; currentBatch = []; } }; const addEntity = (entity)=>{ currentBatch.push(entity); totalCollected++; if (currentBatch.length >= batchSize) { flushBatch(); } }; for (const collection of this.adapter.payload.config.collections.filter(entityHasBlocksField)){ let page = 1; let hasMore = true; while(hasMore){ const { docs, hasNextPage } = await this.adapter.find({ collection: collection.slug, limit: batchSize, page }); for (const doc of docs){ const entity = { id: doc.id, slug: collection.slug, type: 'collection', blocks: collectBlocksToMigrate({ config: this.adapter.payload.config, data: doc, fields: collection.flattenedFields, parentAccessor: [], parentIsLocalized: false }), originalData: doc }; addEntity(entity); } this.adapter.payload.logger.info(`Collected ${docs.length} entries from ${collection.slug} (page ${page})`); hasMore = hasNextPage; page++; } if (collection.versions) { let versionPage = 1; let hasMoreVersions = true; while(hasMoreVersions){ const { docs: versionDocs, hasNextPage: hasNextVersionPage } = await this.adapter.findVersions({ collection: collection.slug, limit: batchSize, page: versionPage }); for (const versionDoc of versionDocs){ const entity = { id: versionDoc.id, slug: collection.slug, type: 'collectionVersion', blocks: collectBlocksToMigrate({ config: this.adapter.payload.config, data: versionDoc, fields: buildVersionCollectionFields(this.adapter.payload.config, collection, true), parentAccessor: [], parentIsLocalized: false }), originalData: versionDoc }; addEntity(entity); } this.adapter.payload.logger.info(`Collected ${versionDocs.length} versions from ${collection.slug} (page ${versionPage})`); hasMoreVersions = hasNextVersionPage; versionPage++; } } } for (const globalConfig of this.adapter.payload.config.globals.filter(entityHasBlocksField)){ const globalDoc = await this.adapter.findGlobal({ slug: globalConfig.slug }); const entity = { slug: globalConfig.slug, type: 'global', blocks: collectBlocksToMigrate({ config: this.adapter.payload.config, data: globalDoc, fields: globalConfig.flattenedFields, parentAccessor: [], parentIsLocalized: false }), originalData: globalDoc }; addEntity(entity); this.adapter.payload.logger.info(`Collected global ${globalConfig.slug}`); if (globalConfig.versions) { let globalVersionPage = 1; let hasMoreGlobalVersions = true; while(hasMoreGlobalVersions){ const { docs: globalVersionDocs, hasNextPage: hasNextGlobalVersionPage } = await this.adapter.findGlobalVersions({ global: globalConfig.slug, limit: batchSize, page: globalVersionPage }); for (const globalVersionDoc of globalVersionDocs){ const entity = { id: globalVersionDoc.id, slug: globalConfig.slug, type: 'globalVersion', blocks: collectBlocksToMigrate({ config: this.adapter.payload.config, data: globalVersionDoc, fields: buildVersionGlobalFields(this.adapter.payload.config, globalConfig, true), parentAccessor: [], parentIsLocalized: false }), originalData: globalVersionDoc }; addEntity(entity); } this.adapter.payload.logger.info(`Collected ${globalVersionDocs.length} versions from global ${globalConfig.slug} (page ${globalVersionPage})`); hasMoreGlobalVersions = hasNextGlobalVersionPage; globalVersionPage++; } } } flushBatch(); this.adapter.payload.logger.info(`Collected total of ${totalCollected} entities across ${batchIndex} batches`); } async getMigrationStatements() { const { generateDrizzleJson, generateMigration } = this.adapter.requireDrizzleKit(); const drizzleJsonBefore = await generateDrizzleJson(this.adapter.schema); this.adapter.blocksAsJSON = true; this.resetSchema(); await this.adapter.init(); const drizzleJsonAfter = await generateDrizzleJson(this.adapter.schema); this.adapter.blocksAsJSON = false; this.resetSchema(); await this.adapter.init(); const statements = await acceptDrizzlePrompts(()=>generateMigration(drizzleJsonBefore, drizzleJsonAfter), { silenceLogs: true }); const sqlExecute = `await db.${this.executeMethod}(` + 'sql`'; return { statements: this.sanitizeStatements({ sqlExecute, statements }), writeDrizzleSnapshot (filePath) { writeFileSync(`${filePath}.json`, JSON.stringify(drizzleJsonAfter, null, 2)); } }; } async migrateEntitiesFromTempFolder(req, options) { const clearBatches = options?.clearBatches ?? true; let totalEntities = 0; let hasEntities = false; for (const entities of this.fetchEntitiesFromJsonBatches()){ hasEntities = true; totalEntities += entities.length; this.adapter.payload.logger.info(`Migrating batch with ${entities.length} entities (total: ${totalEntities})`); await this.migrateEntities(entities, req); } if (!hasEntities) { this.adapter.payload.logger.warn('No entities found in temp folder to migrate'); return; } this.adapter.payload.logger.info(`Completed migration of ${totalEntities} entities from temp folder`); if (clearBatches) { this.cleanupTempFolder(); } else { this.adapter.payload.logger.info(`Temp folder preserved at ${this.tempFolderPath} (clearBatches: false)`); } } setTempFolder(tempFolderPath) { this.tempFolderPath = tempFolderPath; } async updatePayloadConfigFile() { let configPath; try { configPath = findConfig(); } catch { this.adapter.payload.logger.info('updatePayloadConfigFile failed - could not find the payload config. Please set the blocksAsJSON DB adapter property manually to "true"'); return; } const configFile = readFileSync(configPath, 'utf-8'); const ts = await dynamicImport('typescript'); const source = ts.createSourceFile(configPath, configFile, ts.ScriptTarget.ESNext); let hadChanges = false; const result = ts.transform(source, [ (ctx)=>(sourceFile)=>{ const factory = ctx.factory; const visit = (node)=>{ if (ts.isPropertyAssignment(node) && ts.isIdentifier(node.name) && node.name.text === 'db' && ts.isCallExpression(node.initializer) && node.initializer.arguments.length > 0 && ts.isObjectLiteralExpression(node.initializer.arguments[0])) { const call = node.initializer; const obj = node.initializer.arguments[0]; const filteredProps = obj.properties.filter((prop)=>{ if (!ts.isPropertyAssignment(prop)) { return true; } return !(ts.isIdentifier(prop.name) && prop.name.text === 'blocksAsJSON'); }); const newProperty = factory.createPropertyAssignment('blocksAsJSON', factory.createTrue()); hadChanges = true; const newObject = factory.updateObjectLiteralExpression(obj, [ ...filteredProps, newProperty ]); const newCall = factory.updateCallExpression(call, call.expression, call.typeArguments, [ newObject ]); return factory.updatePropertyAssignment(node, node.name, newCall); } return ts.visitEachChild(node, visit, ctx); }; return ts.visitNode(sourceFile, visit); } ]); if (!hadChanges) { this.adapter.payload.logger.info('No changes made to payload config. Set blocksAsJSON to true manually.'); return; } const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed }); let output = printer.printFile(result.transformed[0]); try { // eslint-disable-next-line @typescript-eslint/consistent-type-imports const prettier = await dynamicImport('prettier'); const configPath = await prettier.resolveConfigFile(); const config = configPath ? await prettier.resolveConfig(configPath) : {}; output = await prettier.format(output, { ...config, parser: 'typescript' }); } catch (err) { this.adapter.payload.logger.error({ err, msg: 'Could not format payload config with Prettier' }); } writeFileSync(configPath, output, 'utf-8'); this.adapter.payload.logger.info(`Updated ${configPath} with blocksAsJSON: true`); } } export const createBlocksToJsonMigrator = ({ adapter, executeMethod, sanitizeStatements })=>{ return new BlocksToJsonMigratorImpl(adapter, sanitizeStatements, executeMethod); }; export const getBlocksToJsonMigrator = (payload)=>{ const migrator = payload.db.blocksToJsonMigrator; if (!migrator) { throw new APIError(`blocksToJsonMigrator is not defined for ${payload.db.packageName}`); } return migrator; }; export const buildDynamicPredefinedBlocksToJsonMigration = ({ packageName })=>{ return async ({ filePath, payload })=>{ const migrator = getBlocksToJsonMigrator(payload); const migrationStatements = await migrator.getMigrationStatements(); migrationStatements.writeDrizzleSnapshot(filePath); await migrator.updatePayloadConfigFile(); const upSQL = ` const migrator = getBlocksToJsonMigrator(payload) migrator.setTempFolder(TEMP_FOLDER) await migrator.collectAndSaveEntitiesToBatches(req, { batchSize: BATCH_SIZE }) ${migrationStatements.statements} payload.logger.info("Executed blocks to JSON migration statements.") await migrator.migrateEntitiesFromTempFolder(req, { clearBatches: true }) `; return { imports: ` import { getBlocksToJsonMigrator } from '${packageName}/migration-utils' import { fileURLToPath } from 'url' import path from 'path' const filename = fileURLToPath(import.meta.url) const dirname = path.dirname(filename) // Configure migration options (optional) const BATCH_SIZE = 100 // Number of entities to process per batch const TEMP_FOLDER = path.resolve(dirname, '${TEMP_FOLDER_NAME}') // Folder path to store migration batch `, upSQL }; }; }; //# sourceMappingURL=blocksToJsonMigrator.js.map