Marc Mintel
5397309103
Some checks failed
Build & Deploy / 🔍 Prepare (push) Successful in 20s
Build & Deploy / 🧪 QA (push) Failing after 34s
Build & Deploy / 🏗️ Build (push) Has started running
Build & Deploy / 🚀 Deploy (push) Has been cancelled
Build & Deploy / 🧪 Smoke Test (push) Has been cancelled
Build & Deploy / ⚡ Lighthouse (push) Has been cancelled
Build & Deploy / 🔔 Notify (push) Has been cancelled
587 lines
25 KiB
Plaintext
587 lines
25 KiB
Plaintext
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'fs';
|
|
import path from 'path';
|
|
import { APIError, buildVersionCollectionFields, buildVersionGlobalFields, dynamicImport } from 'payload';
|
|
import { findConfig } from 'payload/node';
|
|
import { fieldShouldBeLocalized } from 'payload/shared';
|
|
import { getTransaction } from './getTransaction.js';
|
|
const DEFAULT_BATCH_SIZE = 100;
|
|
const TEMP_FOLDER_NAME = '.payload-blocks-migration';
|
|
const writeEntitiesToTempFile = (entities, tempFolderPath, batchIndex)=>{
|
|
const filePath = path.join(tempFolderPath, `entities-batch-${batchIndex}.json`);
|
|
writeFileSync(filePath, JSON.stringify(entities, null, 2), 'utf-8');
|
|
};
|
|
const acceptDrizzlePrompts = async (callPrompt, { silenceLogs = false } = {})=>{
|
|
// eslint-disable-next-line @typescript-eslint/unbound-method
|
|
const write = process.stdout.write;
|
|
if (silenceLogs) {
|
|
process.stdout.write = ()=>true;
|
|
}
|
|
const promise = callPrompt();
|
|
const interval = setInterval(()=>process.stdin.emit('keypress', '\n', {
|
|
name: 'return',
|
|
ctrl: false
|
|
}), 25);
|
|
const res = await promise;
|
|
if (silenceLogs) {
|
|
process.stdout.write = write;
|
|
}
|
|
clearInterval(interval);
|
|
return res;
|
|
};
|
|
const entityHasBlocksField = (entity)=>{
|
|
for (const field of entity.flattenedFields){
|
|
if (field.type === 'blocks') {
|
|
return true;
|
|
}
|
|
if ('flattenedFields' in field && entityHasBlocksField({
|
|
flattenedFields: field.flattenedFields
|
|
})) {
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
const collectBlocksToMigrate = ({ config, data, fields, parentAccessor, parentIsLocalized })=>{
|
|
const result = [];
|
|
for (const field of fields){
|
|
switch(field.type){
|
|
case 'array':
|
|
{
|
|
const arrayData = data[field.name];
|
|
if (!Array.isArray(arrayData)) {
|
|
continue;
|
|
}
|
|
if (config.localization && fieldShouldBeLocalized({
|
|
field,
|
|
parentIsLocalized
|
|
})) {
|
|
result.push(...collectBlocksToMigrate({
|
|
config,
|
|
data: arrayData,
|
|
fields: config.localization.localeCodes.map((code)=>({
|
|
...field,
|
|
name: code,
|
|
localized: false
|
|
})),
|
|
parentAccessor: [
|
|
...parentAccessor,
|
|
field.name
|
|
],
|
|
parentIsLocalized: true
|
|
}));
|
|
continue;
|
|
}
|
|
for (const [index, row] of arrayData.entries()){
|
|
result.push(...collectBlocksToMigrate({
|
|
config,
|
|
data: row,
|
|
fields: field.flattenedFields,
|
|
parentAccessor: [
|
|
...parentAccessor,
|
|
field.name,
|
|
index
|
|
],
|
|
parentIsLocalized
|
|
}));
|
|
}
|
|
break;
|
|
}
|
|
case 'blocks':
|
|
{
|
|
result.push({
|
|
data: data[field.name],
|
|
fieldAccessor: [
|
|
...parentAccessor,
|
|
field.name
|
|
]
|
|
});
|
|
break;
|
|
}
|
|
case 'group':
|
|
case 'tab':
|
|
{
|
|
if (config.localization && fieldShouldBeLocalized({
|
|
field,
|
|
parentIsLocalized
|
|
})) {
|
|
result.push(...collectBlocksToMigrate({
|
|
config,
|
|
data: data[field.name],
|
|
fields: config.localization.localeCodes.map((code)=>({
|
|
...field,
|
|
name: code,
|
|
localized: false
|
|
})),
|
|
parentAccessor: [
|
|
...parentAccessor,
|
|
field.name
|
|
],
|
|
parentIsLocalized: true
|
|
}));
|
|
continue;
|
|
}
|
|
result.push(...collectBlocksToMigrate({
|
|
config,
|
|
data: data[field.name],
|
|
fields: field.flattenedFields,
|
|
parentAccessor: [
|
|
...parentAccessor,
|
|
field.name
|
|
],
|
|
parentIsLocalized
|
|
}));
|
|
break;
|
|
}
|
|
default:
|
|
{
|
|
continue;
|
|
}
|
|
}
|
|
}
|
|
return result;
|
|
};
|
|
class BlocksToJsonMigratorImpl {
|
|
adapter;
|
|
sanitizeStatements;
|
|
executeMethod;
|
|
batchSize;
|
|
tempFolderPath;
|
|
constructor(adapter, sanitizeStatements, executeMethod, batchSize){
|
|
this.adapter = adapter;
|
|
this.sanitizeStatements = sanitizeStatements;
|
|
this.executeMethod = executeMethod;
|
|
this.batchSize = batchSize || DEFAULT_BATCH_SIZE;
|
|
this.tempFolderPath = path.resolve(this.adapter.migrationDir, TEMP_FOLDER_NAME);
|
|
}
|
|
ensureTempFolder() {
|
|
if (!existsSync(this.tempFolderPath)) {
|
|
mkdirSync(this.tempFolderPath, {
|
|
recursive: true
|
|
});
|
|
}
|
|
}
|
|
*fetchEntitiesFromJsonBatches() {
|
|
if (!existsSync(this.tempFolderPath)) {
|
|
return;
|
|
}
|
|
const files = readdirSync(this.tempFolderPath).filter((file)=>file.startsWith('entities-batch-') && file.endsWith('.json')).sort((a, b)=>{
|
|
const aNum = parseInt(a.match(/\d+/)?.[0] || '0', 10);
|
|
const bNum = parseInt(b.match(/\d+/)?.[0] || '0', 10);
|
|
return aNum - bNum;
|
|
});
|
|
for (const file of files){
|
|
const filePath = path.join(this.tempFolderPath, file);
|
|
const fileContent = readFileSync(filePath, 'utf-8');
|
|
const batchEntities = JSON.parse(fileContent);
|
|
yield batchEntities;
|
|
}
|
|
}
|
|
async migrateEntities(entities, req) {
|
|
this.adapter.blocksAsJSON = true;
|
|
this.resetSchema();
|
|
await this.adapter.init();
|
|
await this.adapter.connect();
|
|
await this.syncTransactionDrizzleInstance(req);
|
|
const totalEntities = entities.length;
|
|
let processed = 0;
|
|
for (const entity of entities){
|
|
switch(entity.type){
|
|
case 'collection':
|
|
{
|
|
await this.adapter.updateOne({
|
|
collection: entity.slug,
|
|
data: entity.originalData,
|
|
joins: false,
|
|
req,
|
|
where: {
|
|
id: {
|
|
equals: entity.id
|
|
}
|
|
}
|
|
});
|
|
break;
|
|
}
|
|
case 'collectionVersion':
|
|
{
|
|
await this.adapter.updateVersion({
|
|
id: entity.id,
|
|
collection: entity.slug,
|
|
req,
|
|
versionData: entity.originalData
|
|
});
|
|
break;
|
|
}
|
|
case 'global':
|
|
{
|
|
await this.adapter.updateGlobal({
|
|
slug: entity.slug,
|
|
data: entity.originalData,
|
|
req
|
|
});
|
|
break;
|
|
}
|
|
case 'globalVersion':
|
|
{
|
|
await this.adapter.updateGlobalVersion({
|
|
id: entity.id,
|
|
global: entity.slug,
|
|
req,
|
|
versionData: entity.originalData
|
|
});
|
|
break;
|
|
}
|
|
}
|
|
processed++;
|
|
if (processed % this.batchSize === 0 || processed === totalEntities) {
|
|
this.adapter.payload.logger.info(`Migrated ${processed}/${totalEntities} entities (${Math.round(processed / totalEntities * 100)}%)`);
|
|
}
|
|
}
|
|
}
|
|
resetSchema() {
|
|
this.adapter.schema = {};
|
|
this.adapter.tables = {};
|
|
this.adapter.indexes = new Set();
|
|
this.adapter.foreignKeys = new Set();
|
|
this.adapter.relations = {};
|
|
this.adapter.rawTables = {};
|
|
this.adapter.rawRelations = {};
|
|
this.adapter.tableNameMap = new Map();
|
|
this.adapter.enums = {};
|
|
}
|
|
async syncTransactionDrizzleInstance(req) {
|
|
const tsx = await getTransaction(this.adapter, req);
|
|
tsx._ = this.adapter.drizzle._;
|
|
tsx.schema = this.adapter.drizzle._;
|
|
tsx.session.schema = this.adapter.drizzle.session.schema;
|
|
}
|
|
cleanupTempFolder() {
|
|
rmSync(this.tempFolderPath, {
|
|
force: true,
|
|
recursive: true
|
|
});
|
|
this.adapter.payload.logger.info(`Cleaned up temp folder at ${this.tempFolderPath}`);
|
|
}
|
|
async collectAndSaveEntitiesToBatches(req, options) {
|
|
const batchSize = options?.batchSize ?? this.batchSize;
|
|
this.cleanupTempFolder();
|
|
this.ensureTempFolder();
|
|
this.adapter.blocksAsJSON = false;
|
|
this.resetSchema();
|
|
await this.adapter.init();
|
|
await this.adapter.connect();
|
|
// Count total entities to migrate
|
|
let totalExpected = 0;
|
|
for (const collection of this.adapter.payload.config.collections.filter(entityHasBlocksField)){
|
|
const { totalDocs } = await this.adapter.count({
|
|
collection: collection.slug
|
|
});
|
|
totalExpected += totalDocs;
|
|
if (collection.versions) {
|
|
const { totalDocs: totalVersions } = await this.adapter.countVersions({
|
|
collection: collection.slug
|
|
});
|
|
totalExpected += totalVersions;
|
|
}
|
|
}
|
|
for (const globalConfig of this.adapter.payload.config.globals.filter(entityHasBlocksField)){
|
|
totalExpected += 1; // Global itself
|
|
if (globalConfig.versions) {
|
|
const { totalDocs: totalGlobalVersions } = await this.adapter.countGlobalVersions({
|
|
global: globalConfig.slug
|
|
});
|
|
totalExpected += totalGlobalVersions;
|
|
}
|
|
}
|
|
this.adapter.payload.logger.info(`Found ${totalExpected} total entities to collect and save to batches`);
|
|
let batchIndex = 0;
|
|
let currentBatch = [];
|
|
let totalCollected = 0;
|
|
const flushBatch = ()=>{
|
|
if (currentBatch.length > 0) {
|
|
writeEntitiesToTempFile(currentBatch, this.tempFolderPath, batchIndex);
|
|
const percentage = totalExpected > 0 ? Math.round(totalCollected / totalExpected * 100) : 0;
|
|
this.adapter.payload.logger.info(`Saved batch ${batchIndex} with ${currentBatch.length} entities (${totalCollected}/${totalExpected} - ${percentage}%)`);
|
|
batchIndex++;
|
|
currentBatch = [];
|
|
}
|
|
};
|
|
const addEntity = (entity)=>{
|
|
currentBatch.push(entity);
|
|
totalCollected++;
|
|
if (currentBatch.length >= batchSize) {
|
|
flushBatch();
|
|
}
|
|
};
|
|
for (const collection of this.adapter.payload.config.collections.filter(entityHasBlocksField)){
|
|
let page = 1;
|
|
let hasMore = true;
|
|
while(hasMore){
|
|
const { docs, hasNextPage } = await this.adapter.find({
|
|
collection: collection.slug,
|
|
limit: batchSize,
|
|
page
|
|
});
|
|
for (const doc of docs){
|
|
const entity = {
|
|
id: doc.id,
|
|
slug: collection.slug,
|
|
type: 'collection',
|
|
blocks: collectBlocksToMigrate({
|
|
config: this.adapter.payload.config,
|
|
data: doc,
|
|
fields: collection.flattenedFields,
|
|
parentAccessor: [],
|
|
parentIsLocalized: false
|
|
}),
|
|
originalData: doc
|
|
};
|
|
addEntity(entity);
|
|
}
|
|
this.adapter.payload.logger.info(`Collected ${docs.length} entries from ${collection.slug} (page ${page})`);
|
|
hasMore = hasNextPage;
|
|
page++;
|
|
}
|
|
if (collection.versions) {
|
|
let versionPage = 1;
|
|
let hasMoreVersions = true;
|
|
while(hasMoreVersions){
|
|
const { docs: versionDocs, hasNextPage: hasNextVersionPage } = await this.adapter.findVersions({
|
|
collection: collection.slug,
|
|
limit: batchSize,
|
|
page: versionPage
|
|
});
|
|
for (const versionDoc of versionDocs){
|
|
const entity = {
|
|
id: versionDoc.id,
|
|
slug: collection.slug,
|
|
type: 'collectionVersion',
|
|
blocks: collectBlocksToMigrate({
|
|
config: this.adapter.payload.config,
|
|
data: versionDoc,
|
|
fields: buildVersionCollectionFields(this.adapter.payload.config, collection, true),
|
|
parentAccessor: [],
|
|
parentIsLocalized: false
|
|
}),
|
|
originalData: versionDoc
|
|
};
|
|
addEntity(entity);
|
|
}
|
|
this.adapter.payload.logger.info(`Collected ${versionDocs.length} versions from ${collection.slug} (page ${versionPage})`);
|
|
hasMoreVersions = hasNextVersionPage;
|
|
versionPage++;
|
|
}
|
|
}
|
|
}
|
|
for (const globalConfig of this.adapter.payload.config.globals.filter(entityHasBlocksField)){
|
|
const globalDoc = await this.adapter.findGlobal({
|
|
slug: globalConfig.slug
|
|
});
|
|
const entity = {
|
|
slug: globalConfig.slug,
|
|
type: 'global',
|
|
blocks: collectBlocksToMigrate({
|
|
config: this.adapter.payload.config,
|
|
data: globalDoc,
|
|
fields: globalConfig.flattenedFields,
|
|
parentAccessor: [],
|
|
parentIsLocalized: false
|
|
}),
|
|
originalData: globalDoc
|
|
};
|
|
addEntity(entity);
|
|
this.adapter.payload.logger.info(`Collected global ${globalConfig.slug}`);
|
|
if (globalConfig.versions) {
|
|
let globalVersionPage = 1;
|
|
let hasMoreGlobalVersions = true;
|
|
while(hasMoreGlobalVersions){
|
|
const { docs: globalVersionDocs, hasNextPage: hasNextGlobalVersionPage } = await this.adapter.findGlobalVersions({
|
|
global: globalConfig.slug,
|
|
limit: batchSize,
|
|
page: globalVersionPage
|
|
});
|
|
for (const globalVersionDoc of globalVersionDocs){
|
|
const entity = {
|
|
id: globalVersionDoc.id,
|
|
slug: globalConfig.slug,
|
|
type: 'globalVersion',
|
|
blocks: collectBlocksToMigrate({
|
|
config: this.adapter.payload.config,
|
|
data: globalVersionDoc,
|
|
fields: buildVersionGlobalFields(this.adapter.payload.config, globalConfig, true),
|
|
parentAccessor: [],
|
|
parentIsLocalized: false
|
|
}),
|
|
originalData: globalVersionDoc
|
|
};
|
|
addEntity(entity);
|
|
}
|
|
this.adapter.payload.logger.info(`Collected ${globalVersionDocs.length} versions from global ${globalConfig.slug} (page ${globalVersionPage})`);
|
|
hasMoreGlobalVersions = hasNextGlobalVersionPage;
|
|
globalVersionPage++;
|
|
}
|
|
}
|
|
}
|
|
flushBatch();
|
|
this.adapter.payload.logger.info(`Collected total of ${totalCollected} entities across ${batchIndex} batches`);
|
|
}
|
|
async getMigrationStatements() {
|
|
const { generateDrizzleJson, generateMigration } = this.adapter.requireDrizzleKit();
|
|
const drizzleJsonBefore = await generateDrizzleJson(this.adapter.schema);
|
|
this.adapter.blocksAsJSON = true;
|
|
this.resetSchema();
|
|
await this.adapter.init();
|
|
const drizzleJsonAfter = await generateDrizzleJson(this.adapter.schema);
|
|
this.adapter.blocksAsJSON = false;
|
|
this.resetSchema();
|
|
await this.adapter.init();
|
|
const statements = await acceptDrizzlePrompts(()=>generateMigration(drizzleJsonBefore, drizzleJsonAfter), {
|
|
silenceLogs: true
|
|
});
|
|
const sqlExecute = `await db.${this.executeMethod}(` + 'sql`';
|
|
return {
|
|
statements: this.sanitizeStatements({
|
|
sqlExecute,
|
|
statements
|
|
}),
|
|
writeDrizzleSnapshot (filePath) {
|
|
writeFileSync(`${filePath}.json`, JSON.stringify(drizzleJsonAfter, null, 2));
|
|
}
|
|
};
|
|
}
|
|
async migrateEntitiesFromTempFolder(req, options) {
|
|
const clearBatches = options?.clearBatches ?? true;
|
|
let totalEntities = 0;
|
|
let hasEntities = false;
|
|
for (const entities of this.fetchEntitiesFromJsonBatches()){
|
|
hasEntities = true;
|
|
totalEntities += entities.length;
|
|
this.adapter.payload.logger.info(`Migrating batch with ${entities.length} entities (total: ${totalEntities})`);
|
|
await this.migrateEntities(entities, req);
|
|
}
|
|
if (!hasEntities) {
|
|
this.adapter.payload.logger.warn('No entities found in temp folder to migrate');
|
|
return;
|
|
}
|
|
this.adapter.payload.logger.info(`Completed migration of ${totalEntities} entities from temp folder`);
|
|
if (clearBatches) {
|
|
this.cleanupTempFolder();
|
|
} else {
|
|
this.adapter.payload.logger.info(`Temp folder preserved at ${this.tempFolderPath} (clearBatches: false)`);
|
|
}
|
|
}
|
|
setTempFolder(tempFolderPath) {
|
|
this.tempFolderPath = tempFolderPath;
|
|
}
|
|
async updatePayloadConfigFile() {
|
|
let configPath;
|
|
try {
|
|
configPath = findConfig();
|
|
} catch {
|
|
this.adapter.payload.logger.info('updatePayloadConfigFile failed - could not find the payload config. Please set the blocksAsJSON DB adapter property manually to "true"');
|
|
return;
|
|
}
|
|
const configFile = readFileSync(configPath, 'utf-8');
|
|
const ts = await dynamicImport('typescript');
|
|
const source = ts.createSourceFile(configPath, configFile, ts.ScriptTarget.ESNext);
|
|
let hadChanges = false;
|
|
const result = ts.transform(source, [
|
|
(ctx)=>(sourceFile)=>{
|
|
const factory = ctx.factory;
|
|
const visit = (node)=>{
|
|
if (ts.isPropertyAssignment(node) && ts.isIdentifier(node.name) && node.name.text === 'db' && ts.isCallExpression(node.initializer) && node.initializer.arguments.length > 0 && ts.isObjectLiteralExpression(node.initializer.arguments[0])) {
|
|
const call = node.initializer;
|
|
const obj = node.initializer.arguments[0];
|
|
const filteredProps = obj.properties.filter((prop)=>{
|
|
if (!ts.isPropertyAssignment(prop)) {
|
|
return true;
|
|
}
|
|
return !(ts.isIdentifier(prop.name) && prop.name.text === 'blocksAsJSON');
|
|
});
|
|
const newProperty = factory.createPropertyAssignment('blocksAsJSON', factory.createTrue());
|
|
hadChanges = true;
|
|
const newObject = factory.updateObjectLiteralExpression(obj, [
|
|
...filteredProps,
|
|
newProperty
|
|
]);
|
|
const newCall = factory.updateCallExpression(call, call.expression, call.typeArguments, [
|
|
newObject
|
|
]);
|
|
return factory.updatePropertyAssignment(node, node.name, newCall);
|
|
}
|
|
return ts.visitEachChild(node, visit, ctx);
|
|
};
|
|
return ts.visitNode(sourceFile, visit);
|
|
}
|
|
]);
|
|
if (!hadChanges) {
|
|
this.adapter.payload.logger.info('No changes made to payload config. Set blocksAsJSON to true manually.');
|
|
return;
|
|
}
|
|
const printer = ts.createPrinter({
|
|
newLine: ts.NewLineKind.LineFeed
|
|
});
|
|
let output = printer.printFile(result.transformed[0]);
|
|
try {
|
|
// eslint-disable-next-line @typescript-eslint/consistent-type-imports
|
|
const prettier = await dynamicImport('prettier');
|
|
const configPath = await prettier.resolveConfigFile();
|
|
const config = configPath ? await prettier.resolveConfig(configPath) : {};
|
|
output = await prettier.format(output, {
|
|
...config,
|
|
parser: 'typescript'
|
|
});
|
|
} catch (err) {
|
|
this.adapter.payload.logger.error({
|
|
err,
|
|
msg: 'Could not format payload config with Prettier'
|
|
});
|
|
}
|
|
writeFileSync(configPath, output, 'utf-8');
|
|
this.adapter.payload.logger.info(`Updated ${configPath} with blocksAsJSON: true`);
|
|
}
|
|
}
|
|
export const createBlocksToJsonMigrator = ({ adapter, executeMethod, sanitizeStatements })=>{
|
|
return new BlocksToJsonMigratorImpl(adapter, sanitizeStatements, executeMethod);
|
|
};
|
|
export const getBlocksToJsonMigrator = (payload)=>{
|
|
const migrator = payload.db.blocksToJsonMigrator;
|
|
if (!migrator) {
|
|
throw new APIError(`blocksToJsonMigrator is not defined for ${payload.db.packageName}`);
|
|
}
|
|
return migrator;
|
|
};
|
|
export const buildDynamicPredefinedBlocksToJsonMigration = ({ packageName })=>{
|
|
return async ({ filePath, payload })=>{
|
|
const migrator = getBlocksToJsonMigrator(payload);
|
|
const migrationStatements = await migrator.getMigrationStatements();
|
|
migrationStatements.writeDrizzleSnapshot(filePath);
|
|
await migrator.updatePayloadConfigFile();
|
|
const upSQL = `
|
|
const migrator = getBlocksToJsonMigrator(payload)
|
|
migrator.setTempFolder(TEMP_FOLDER)
|
|
await migrator.collectAndSaveEntitiesToBatches(req, { batchSize: BATCH_SIZE })
|
|
|
|
${migrationStatements.statements}
|
|
payload.logger.info("Executed blocks to JSON migration statements.")
|
|
|
|
await migrator.migrateEntitiesFromTempFolder(req, { clearBatches: true })
|
|
`;
|
|
return {
|
|
imports: `
|
|
import { getBlocksToJsonMigrator } from '${packageName}/migration-utils'
|
|
import { fileURLToPath } from 'url'
|
|
import path from 'path'
|
|
|
|
const filename = fileURLToPath(import.meta.url)
|
|
const dirname = path.dirname(filename)
|
|
|
|
// Configure migration options (optional)
|
|
const BATCH_SIZE = 100 // Number of entities to process per batch
|
|
const TEMP_FOLDER = path.resolve(dirname, '${TEMP_FOLDER_NAME}') // Folder path to store migration batch
|
|
`,
|
|
upSQL
|
|
};
|
|
};
|
|
};
|
|
|
|
//# sourceMappingURL=blocksToJsonMigrator.js.map |