import fs from 'fs/promises'; import path from 'path'; import { resolveCodec, getFormatExtension } from '../format/index.js'; import SourceFileScanner from '../source/SourceFileScanner.js'; import SourceFileWatcher from '../source/SourceFileWatcher.js'; import { getDefaultProjectRoot, normalizePathToPosix, compareReferences } from '../utils.js'; import CatalogLocales from './CatalogLocales.js'; import CatalogPersister from './CatalogPersister.js'; import SaveScheduler from './SaveScheduler.js'; class CatalogManager { /** * The source of truth for which messages are used. * NOTE: Should be mutated in place to keep `messagesById` and `messagesByFile` in sync. */ messagesByFile = new Map(); /** * Fast lookup for messages by ID across all files, * contains the same messages as `messagesByFile`. * NOTE: Should be mutated in place to keep `messagesById` and `messagesByFile` in sync. */ messagesById = new Map(); /** * This potentially also includes outdated ones that were initially available, * but are not used anymore. This allows to restore them if they are used again. **/ translationsByTargetLocale = new Map(); lastWriteByLocale = new Map(); // Cached instances // Resolves when all catalogs are loaded // Resolves when the initial project scan and processing is complete constructor(config, opts) { this.config = config; this.saveScheduler = new SaveScheduler(50); this.projectRoot = opts.projectRoot ?? getDefaultProjectRoot(); this.isDevelopment = opts.isDevelopment ?? false; this.extractor = opts.extractor; if (this.isDevelopment) { // We kick this off as early as possible, so we get notified about changes // that happen during the initial project scan (while awaiting it to // complete though) this.sourceWatcher = new SourceFileWatcher(this.getSrcPaths(), this.handleFileEvents.bind(this)); void this.sourceWatcher.start(); } } async getCodec() { if (!this.codec) { this.codec = await resolveCodec(this.config.messages.format, this.projectRoot); } return this.codec; } async getPersister() { if (this.persister) { return this.persister; } else { this.persister = new CatalogPersister({ messagesPath: this.config.messages.path, codec: await this.getCodec(), extension: getFormatExtension(this.config.messages.format) }); return this.persister; } } getCatalogLocales() { if (this.catalogLocales) { return this.catalogLocales; } else { const messagesDir = path.join(this.projectRoot, this.config.messages.path); this.catalogLocales = new CatalogLocales({ messagesDir, sourceLocale: this.config.sourceLocale, extension: getFormatExtension(this.config.messages.format), locales: this.config.messages.locales }); return this.catalogLocales; } } async getTargetLocales() { return this.getCatalogLocales().getTargetLocales(); } getSrcPaths() { return (Array.isArray(this.config.srcPath) ? this.config.srcPath : [this.config.srcPath]).map(srcPath => path.join(this.projectRoot, srcPath)); } async loadMessages() { const sourceDiskMessages = await this.loadSourceMessages(); this.loadCatalogsPromise = this.loadTargetMessages(); await this.loadCatalogsPromise; this.scanCompletePromise = (async () => { const sourceFiles = await SourceFileScanner.getSourceFiles(this.getSrcPaths()); await Promise.all(Array.from(sourceFiles).map(async filePath => this.processFile(filePath))); this.mergeSourceDiskMetadata(sourceDiskMessages); })(); await this.scanCompletePromise; if (this.isDevelopment) { const catalogLocales = this.getCatalogLocales(); catalogLocales.subscribeLocalesChange(this.onLocalesChange); } } async loadSourceMessages() { // Load source catalog to hydrate metadata (e.g. flags) later without // treating catalog entries as source of truth. const diskMessages = await this.loadLocaleMessages(this.config.sourceLocale); const byId = new Map(); for (const diskMessage of diskMessages) { byId.set(diskMessage.id, diskMessage); } return byId; } async loadLocaleMessages(locale) { const persister = await this.getPersister(); const messages = await persister.read(locale); const fileTime = await persister.getLastModified(locale); this.lastWriteByLocale.set(locale, fileTime); return messages; } async loadTargetMessages() { const targetLocales = await this.getTargetLocales(); await Promise.all(targetLocales.map(locale => this.reloadLocaleCatalog(locale))); } async reloadLocaleCatalog(locale) { const diskMessages = await this.loadLocaleMessages(locale); if (locale === this.config.sourceLocale) { // For source: Merge additional properties like flags for (const diskMessage of diskMessages) { const prev = this.messagesById.get(diskMessage.id); if (prev) { // Mutate the existing object instead of creating a copy // to keep messagesById and messagesByFile in sync. // Unknown properties (like flags): disk wins // Known properties: existing (from extraction) wins for (const key of Object.keys(diskMessage)) { if (!['id', 'message', 'description', 'references'].includes(key)) { // For unknown properties (like flags), disk wins prev[key] = diskMessage[key]; } } } } } else { // For target: disk wins completely, BUT preserve existing translations // if we read empty (likely a write in progress by an external tool // that causes the file to temporarily be empty) const existingTranslations = this.translationsByTargetLocale.get(locale); const hasExistingTranslations = existingTranslations && existingTranslations.size > 0; if (diskMessages.length > 0) { // We got content from disk, replace with it const translations = new Map(); for (const message of diskMessages) { translations.set(message.id, message); } this.translationsByTargetLocale.set(locale, translations); } else if (hasExistingTranslations) ; else { // We read empty and have no existing translations const translations = new Map(); this.translationsByTargetLocale.set(locale, translations); } } } mergeSourceDiskMetadata(diskMessages) { for (const [id, diskMessage] of diskMessages) { const existing = this.messagesById.get(id); if (!existing) continue; // Mutate the existing object instead of creating a copy. // This keeps `messagesById` and `messagesByFile` in sync since // they reference the same object instance. for (const key of Object.keys(diskMessage)) { if (existing[key] == null) { existing[key] = diskMessage[key]; } } } } async processFile(absoluteFilePath) { let messages = []; try { const content = await fs.readFile(absoluteFilePath, 'utf8'); let extraction; try { extraction = await this.extractor.extract(absoluteFilePath, content); } catch { return false; } messages = extraction.messages; } catch (err) { if (err.code !== 'ENOENT') { throw err; } // ENOENT -> treat as no messages } const prevFileMessages = this.messagesByFile.get(absoluteFilePath); const relativeFilePath = normalizePathToPosix(path.relative(this.projectRoot, absoluteFilePath)); // Init with all previous ones const idsToRemove = Array.from(prevFileMessages?.keys() ?? []); // Replace existing messages with new ones const fileMessages = new Map(); for (let message of messages) { const prevMessage = this.messagesById.get(message.id); // Merge with previous message if it exists if (prevMessage) { message = { ...message }; if (message.references) { message.references = this.mergeReferences(prevMessage.references ?? [], relativeFilePath, message.references); } // Merge other properties like description, or unknown // attributes like flags that are opaque to us for (const key of Object.keys(prevMessage)) { if (message[key] == null) { message[key] = prevMessage[key]; } } } this.messagesById.set(message.id, message); fileMessages.set(message.id, message); // This message continues to exist in this file const index = idsToRemove.indexOf(message.id); if (index !== -1) idsToRemove.splice(index, 1); } // Clean up removed messages from `messagesById` idsToRemove.forEach(id => { const message = this.messagesById.get(id); if (!message) return; const hasOtherReferences = message.references?.some(ref => ref.path !== relativeFilePath); if (!hasOtherReferences) { // No other references, delete the message entirely this.messagesById.delete(id); } else { // Message is used elsewhere, remove this file from references // Mutate the existing object to keep `messagesById` and `messagesByFile` in sync message.references = message.references?.filter(ref => ref.path !== relativeFilePath); } }); // Update the stored messages if (messages.length > 0) { this.messagesByFile.set(absoluteFilePath, fileMessages); } else { this.messagesByFile.delete(absoluteFilePath); } const changed = this.haveMessagesChangedForFile(prevFileMessages, fileMessages); return changed; } mergeReferences(existing, currentFilePath, currentFileRefs) { // Keep refs from other files, replace all refs from the current file const otherFileRefs = existing.filter(ref => ref.path !== currentFilePath); const merged = [...otherFileRefs, ...currentFileRefs]; return merged.sort(compareReferences); } haveMessagesChangedForFile(beforeMessages, afterMessages) { // If one exists and the other doesn't, there's a change if (!beforeMessages) { return afterMessages.size > 0; } // Different sizes means changes if (beforeMessages.size !== afterMessages.size) { return true; } // Check differences in beforeMessages vs afterMessages for (const [id, msg1] of beforeMessages) { const msg2 = afterMessages.get(id); if (!msg2 || !this.areMessagesEqual(msg1, msg2)) { return true; // Early exit on first difference } } return false; } areMessagesEqual(msg1, msg2) { // Note: We intentionally don't compare references here. // References are aggregated metadata from multiple files and comparing // them would cause false positives due to parallel extraction order. return msg1.id === msg2.id && msg1.message === msg2.message && msg1.description === msg2.description; } async save() { return this.saveScheduler.schedule(() => this.saveImpl()); } async saveImpl() { await this.saveLocale(this.config.sourceLocale); const targetLocales = await this.getTargetLocales(); await Promise.all(targetLocales.map(locale => this.saveLocale(locale))); } async saveLocale(locale) { await this.loadCatalogsPromise; const messages = Array.from(this.messagesById.values()); const persister = await this.getPersister(); const isSourceLocale = locale === this.config.sourceLocale; // Check if file was modified externally (poll-at-save is cheaper than // watchers here since stat() is fast and avoids continuous overhead) const lastWriteTime = this.lastWriteByLocale.get(locale); const currentFileTime = await persister.getLastModified(locale); if (currentFileTime && lastWriteTime && currentFileTime > lastWriteTime) { await this.reloadLocaleCatalog(locale); } const localeMessages = isSourceLocale ? this.messagesById : this.translationsByTargetLocale.get(locale); const messagesToPersist = messages.map(message => { const localeMessage = localeMessages?.get(message.id); return { ...localeMessage, id: message.id, description: message.description, references: message.references, message: isSourceLocale ? message.message : localeMessage?.message ?? '' }; }); await persister.write(messagesToPersist, { locale, sourceMessagesById: this.messagesById }); // Update timestamps const newTime = await persister.getLastModified(locale); this.lastWriteByLocale.set(locale, newTime); } onLocalesChange = async params => { // Chain to existing promise this.loadCatalogsPromise = Promise.all([this.loadCatalogsPromise, ...params.added.map(locale => this.reloadLocaleCatalog(locale))]); for (const locale of params.added) { await this.saveLocale(locale); } for (const locale of params.removed) { this.translationsByTargetLocale.delete(locale); this.lastWriteByLocale.delete(locale); } }; async handleFileEvents(events) { if (this.loadCatalogsPromise) { await this.loadCatalogsPromise; } // Wait for initial scan to complete to avoid race conditions if (this.scanCompletePromise) { await this.scanCompletePromise; } let changed = false; const expandedEvents = await this.sourceWatcher.expandDirectoryDeleteEvents(events, Array.from(this.messagesByFile.keys())); for (const event of expandedEvents) { const hasChanged = await this.processFile(event.path); changed ||= hasChanged; } if (changed) { await this.save(); } } [Symbol.dispose]() { this.sourceWatcher?.stop(); this.sourceWatcher = undefined; this.saveScheduler[Symbol.dispose](); if (this.catalogLocales && this.isDevelopment) { this.catalogLocales.unsubscribeLocalesChange(this.onLocalesChange); } } } export { CatalogManager as default };