Some checks failed
Build & Deploy / 🔍 Prepare (push) Successful in 20s
Build & Deploy / 🧪 QA (push) Failing after 34s
Build & Deploy / 🏗️ Build (push) Has started running
Build & Deploy / 🚀 Deploy (push) Has been cancelled
Build & Deploy / 🧪 Smoke Test (push) Has been cancelled
Build & Deploy / ⚡ Lighthouse (push) Has been cancelled
Build & Deploy / 🔔 Notify (push) Has been cancelled
350 lines
13 KiB
Plaintext
350 lines
13 KiB
Plaintext
import { memoize } from "@formatjs/fast-memoize";
|
||
import { data as jsonData } from "./languageMatching.js";
|
||
import { regions } from "./regions.generated.js";
|
||
export const UNICODE_EXTENSION_SEQUENCE_REGEX = /-u(?:-[0-9a-z]{2,8})+/gi;
|
||
/**
|
||
* Asserts that a condition is true, throwing an error if it is not.
|
||
* Used for runtime validation and type narrowing.
|
||
*
|
||
* @param condition - The condition to check
|
||
* @param message - Error message if condition is false
|
||
* @param Err - Error constructor to use (defaults to Error)
|
||
* @throws {Error} When condition is false
|
||
*
|
||
* @example
|
||
* ```ts
|
||
* invariant(locale !== undefined, 'Locale must be defined')
|
||
* // locale is now narrowed to non-undefined type
|
||
* ```
|
||
*/
|
||
export function invariant(condition, message, Err = Error) {
|
||
if (!condition) {
|
||
throw new Err(message);
|
||
}
|
||
}
|
||
// This is effectively 2 languages in 2 different regions in the same cluster
|
||
const DEFAULT_MATCHING_THRESHOLD = 838;
|
||
let PROCESSED_DATA;
|
||
function processData() {
|
||
if (!PROCESSED_DATA) {
|
||
const paradigmLocales = jsonData.supplemental.languageMatching["written-new"][0]?.paradigmLocales?._locales.split(" ");
|
||
const matchVariables = jsonData.supplemental.languageMatching["written-new"].slice(1, 5);
|
||
const data = jsonData.supplemental.languageMatching["written-new"].slice(5);
|
||
const matches = data.map((d) => {
|
||
const key = Object.keys(d)[0];
|
||
const value = d[key];
|
||
return {
|
||
supported: key,
|
||
desired: value._desired,
|
||
distance: +value._distance,
|
||
oneway: value.oneway === "true" ? true : false
|
||
};
|
||
}, {});
|
||
PROCESSED_DATA = {
|
||
matches,
|
||
matchVariables: matchVariables.reduce((all, d) => {
|
||
const key = Object.keys(d)[0];
|
||
const value = d[key];
|
||
all[key.slice(1)] = value._value.split("+");
|
||
return all;
|
||
}, {}),
|
||
paradigmLocales: [...paradigmLocales, ...paradigmLocales.map((l) => new Intl.Locale(l.replace(/_/g, "-")).maximize().toString())]
|
||
};
|
||
}
|
||
return PROCESSED_DATA;
|
||
}
|
||
function isMatched(locale, languageMatchInfoLocale, matchVariables) {
|
||
const [language, script, region] = languageMatchInfoLocale.split("-");
|
||
let matches = true;
|
||
if (region && region[0] === "$") {
|
||
const shouldInclude = region[1] !== "!";
|
||
const matchRegions = shouldInclude ? matchVariables[region.slice(1)] : matchVariables[region.slice(2)];
|
||
const expandedMatchedRegions = matchRegions.map((r) => regions[r] || [r]).reduce((all, list) => [...all, ...list], []);
|
||
matches &&= !(expandedMatchedRegions.indexOf(locale.region || "") > -1 != shouldInclude);
|
||
} else {
|
||
matches &&= locale.region ? region === "*" || region === locale.region : true;
|
||
}
|
||
matches &&= locale.script ? script === "*" || script === locale.script : true;
|
||
matches &&= locale.language ? language === "*" || language === locale.language : true;
|
||
return matches;
|
||
}
|
||
function serializeLSR(lsr) {
|
||
return [
|
||
lsr.language,
|
||
lsr.script,
|
||
lsr.region
|
||
].filter(Boolean).join("-");
|
||
}
|
||
function findMatchingDistanceForLSR(desired, supported, data) {
|
||
for (const d of data.matches) {
|
||
let matches = isMatched(desired, d.desired, data.matchVariables) && isMatched(supported, d.supported, data.matchVariables);
|
||
if (!d.oneway && !matches) {
|
||
matches = isMatched(desired, d.supported, data.matchVariables) && isMatched(supported, d.desired, data.matchVariables);
|
||
}
|
||
if (matches) {
|
||
const distance = d.distance * 10;
|
||
if (data.paradigmLocales.indexOf(serializeLSR(desired)) > -1 != data.paradigmLocales.indexOf(serializeLSR(supported)) > -1) {
|
||
return distance - 1;
|
||
}
|
||
return distance;
|
||
}
|
||
}
|
||
throw new Error("No matching distance found");
|
||
}
|
||
function findMatchingDistanceImpl(desired, supported) {
|
||
const desiredLocale = new Intl.Locale(desired).maximize();
|
||
const supportedLocale = new Intl.Locale(supported).maximize();
|
||
const desiredLSR = {
|
||
language: desiredLocale.language,
|
||
script: desiredLocale.script || "",
|
||
region: desiredLocale.region || ""
|
||
};
|
||
const supportedLSR = {
|
||
language: supportedLocale.language,
|
||
script: supportedLocale.script || "",
|
||
region: supportedLocale.region || ""
|
||
};
|
||
let matchingDistance = 0;
|
||
const data = processData();
|
||
if (desiredLSR.language !== supportedLSR.language) {
|
||
matchingDistance += findMatchingDistanceForLSR({
|
||
language: desiredLocale.language,
|
||
script: "",
|
||
region: ""
|
||
}, {
|
||
language: supportedLocale.language,
|
||
script: "",
|
||
region: ""
|
||
}, data);
|
||
}
|
||
if (desiredLSR.script !== supportedLSR.script) {
|
||
matchingDistance += findMatchingDistanceForLSR({
|
||
language: desiredLocale.language,
|
||
script: desiredLSR.script,
|
||
region: ""
|
||
}, {
|
||
language: supportedLocale.language,
|
||
script: supportedLSR.script,
|
||
region: ""
|
||
}, data);
|
||
}
|
||
if (desiredLSR.region !== supportedLSR.region) {
|
||
matchingDistance += findMatchingDistanceForLSR(desiredLSR, supportedLSR, data);
|
||
}
|
||
return matchingDistance;
|
||
}
|
||
/**
|
||
* Calculates the matching distance between two locales using the CLDR Enhanced Language Matching algorithm.
|
||
* This function is memoized for performance, as distance calculations are expensive.
|
||
*
|
||
* The distance represents how "far apart" two locales are, with 0 being identical (after maximization).
|
||
* Distances are calculated based on Language-Script-Region (LSR) differences using CLDR data.
|
||
*
|
||
* @param desired - The desired locale (e.g., "en-US")
|
||
* @param supported - The supported locale to compare against (e.g., "en-GB")
|
||
* @returns The calculated distance between the locales
|
||
*
|
||
* @example
|
||
* ```ts
|
||
* findMatchingDistance('en-US', 'en-US') // 0 - identical
|
||
* findMatchingDistance('en-US', 'en-GB') // 40 - same language/script, different region
|
||
* findMatchingDistance('es-CO', 'es-419') // 39 - regional variant
|
||
* findMatchingDistance('en', 'fr') // 840 - completely different languages
|
||
* ```
|
||
*
|
||
* @see https://unicode.org/reports/tr35/#EnhancedLanguageMatching
|
||
*/
|
||
export const findMatchingDistance = memoize(findMatchingDistanceImpl, { serializer: (args) => `${args[0]}|${args[1]}` });
|
||
/**
|
||
* Generates fallback candidates by progressively removing subtags
|
||
* e.g., "en-US" -> ["en-US", "en"]
|
||
* "zh-Hans-CN" -> ["zh-Hans-CN", "zh-Hans", "zh"]
|
||
*/
|
||
function getFallbackCandidates(locale) {
|
||
const candidates = [];
|
||
let current = locale;
|
||
while (current) {
|
||
candidates.push(current);
|
||
const lastDash = current.lastIndexOf("-");
|
||
if (lastDash === -1) break;
|
||
current = current.substring(0, lastDash);
|
||
}
|
||
return candidates;
|
||
}
|
||
/**
|
||
* Finds the best locale match using a three-tier optimization hierarchy.
|
||
*
|
||
* ## Three-Tier Matching Algorithm:
|
||
*
|
||
* **Tier 1 - Fast Path** (O(n)): Exact string matching via Set lookup
|
||
* - Example: 'en' matches 'en' exactly → distance 0
|
||
* - Solves #4936: 48x faster than baseline (12ms vs 610ms with 700+ locales)
|
||
*
|
||
* **Tier 2 - Fallback Path** (O(k×n)): Maximization + progressive subtag removal
|
||
* - Maximizes requested locale, then removes subtags right-to-left
|
||
* - Example: "zh-TW" → "zh-Hant-TW" → ["zh-Hant-TW", "zh-Hant", "zh"]
|
||
* - Distance: 0 for maximized match, 10 per removed subtag + position penalty
|
||
* - 40-50x faster than full UTS #35, handles 99% of real-world cases correctly
|
||
*
|
||
* **Tier 3 - Slow Path** (O(n×m), memoized): Full UTS #35 CLDR matching
|
||
* - Calculates Language-Script-Region distances using CLDR data
|
||
* - Handles complex cases like cross-script matching (sr-Cyrl ↔ sr-Latn)
|
||
* - Only used when Tiers 1 & 2 find no match
|
||
* - Still 6x faster than baseline due to memoization
|
||
*
|
||
* ## Performance Impact of Maximization:
|
||
*
|
||
* While Tier 2 now calls `Intl.Locale().maximize()` once per requested locale,
|
||
* this is still much faster than Tier 3's full distance calculation:
|
||
* - Tier 1: ~12ms (exact match, no maximization)
|
||
* - Tier 2: ~13-15ms (maximization + fallback)
|
||
* - Tier 3: ~100ms+ (full UTS #35 with all supported locales)
|
||
*
|
||
* @param requestedLocales - Locale identifiers in preference order
|
||
* @param supportedLocales - Available locale identifiers
|
||
* @param threshold - Maximum distance (default: 838, from CLDR)
|
||
* @returns Matching result with distances
|
||
*
|
||
* @example
|
||
* ```ts
|
||
* // Tier 1: Exact match
|
||
* findBestMatch(['en'], ['en', 'fr'])
|
||
* // → { matchedSupportedLocale: 'en', distances: { en: { en: 0 } } }
|
||
*
|
||
* // Tier 2: Fallback with maximization
|
||
* findBestMatch(['zh-TW'], ['zh-Hant'])
|
||
* // → zh-TW maximizes to zh-Hant-TW, falls back to zh-Hant (distance 0)
|
||
*
|
||
* findBestMatch(['en-US'], ['en'])
|
||
* // → en-US maximizes to en-Latn-US, falls back to en (distance 10)
|
||
*
|
||
* // Tier 3: Full calculation
|
||
* findBestMatch(['en-XZ'], ['ja', 'ko'])
|
||
* // → No fallback match, uses UTS #35 to find closest match
|
||
* ```
|
||
*
|
||
* @see https://unicode.org/reports/tr35/#EnhancedLanguageMatching
|
||
* @see https://github.com/formatjs/formatjs/issues/4936
|
||
*/
|
||
// WeakMap to cache canonicalized supported locales arrays
|
||
const canonicalizedSupportedCache = new WeakMap();
|
||
export function findBestMatch(requestedLocales, supportedLocales, threshold = DEFAULT_MATCHING_THRESHOLD) {
|
||
let lowestDistance = Infinity;
|
||
let result = {
|
||
matchedDesiredLocale: "",
|
||
distances: {}
|
||
};
|
||
// Get or compute canonicalized supported locales (one by one to preserve indices)
|
||
let canonicalizedSupportedLocales = canonicalizedSupportedCache.get(supportedLocales);
|
||
if (!canonicalizedSupportedLocales) {
|
||
canonicalizedSupportedLocales = supportedLocales.map((locale) => {
|
||
try {
|
||
const canonical = Intl.getCanonicalLocales([locale]);
|
||
return canonical[0] || locale;
|
||
} catch {
|
||
return locale;
|
||
}
|
||
});
|
||
canonicalizedSupportedCache.set(supportedLocales, canonicalizedSupportedLocales);
|
||
}
|
||
const supportedSet = new Set(canonicalizedSupportedLocales);
|
||
// === TIER 1: FAST PATH - Exact Match ===
|
||
// Check for exact matches in ALL requested locales
|
||
// This is the fastest path and handles the majority of real-world cases
|
||
for (let i = 0; i < requestedLocales.length; i++) {
|
||
const desired = requestedLocales[i];
|
||
if (supportedSet.has(desired)) {
|
||
const distance = 0 + i * 40;
|
||
result.distances[desired] = { [desired]: distance };
|
||
if (distance < lowestDistance) {
|
||
lowestDistance = distance;
|
||
result.matchedDesiredLocale = desired;
|
||
result.matchedSupportedLocale = desired;
|
||
}
|
||
// Only return immediately if this is the first requested locale (distance=0)
|
||
// Otherwise, continue checking for potentially better matches
|
||
if (i === 0) {
|
||
return result;
|
||
}
|
||
}
|
||
}
|
||
// If we found an exact match in Tier 1 (but not for first locale), check Tier 2
|
||
// to see if there's a better fallback match with lower distance
|
||
// If no exact match found, Tier 2 will find fallback matches
|
||
// === TIER 2: FALLBACK PATH - Maximization + Progressive Subtag Removal ===
|
||
// Try maximization-based matching before resorting to expensive Tier 3
|
||
// This handles cases like zh-TW → zh-Hant efficiently
|
||
for (let i = 0; i < requestedLocales.length; i++) {
|
||
const desired = requestedLocales[i];
|
||
// Maximize then fallback (for linguistic accuracy like zh-TW → zh-Hant)
|
||
try {
|
||
const maximized = new Intl.Locale(desired).maximize().toString();
|
||
if (maximized !== desired) {
|
||
const maximizedCandidates = getFallbackCandidates(maximized);
|
||
for (let j = 0; j < maximizedCandidates.length; j++) {
|
||
const candidate = maximizedCandidates[j];
|
||
if (candidate === desired) continue;
|
||
if (supportedSet.has(candidate)) {
|
||
// Check if candidate also maximizes to the same form
|
||
// e.g., zh-TW → zh-Hant-TW and zh-Hant → zh-Hant-TW (distance 0)
|
||
// but es-co → es-Latn-CO and es → es-Latn-ES (distance 10)
|
||
let distance;
|
||
try {
|
||
const candidateMaximized = new Intl.Locale(candidate).maximize().toString();
|
||
distance = candidateMaximized === maximized ? 0 + i * 40 : j * 10 + i * 40;
|
||
} catch {
|
||
distance = j * 10 + i * 40;
|
||
}
|
||
if (!result.distances[desired]) {
|
||
result.distances[desired] = {};
|
||
}
|
||
result.distances[desired][candidate] = distance;
|
||
if (distance < lowestDistance) {
|
||
lowestDistance = distance;
|
||
result.matchedDesiredLocale = desired;
|
||
result.matchedSupportedLocale = candidate;
|
||
}
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
} catch {}
|
||
}
|
||
// If Tier 2 found a perfect maximized match (distance 0), return immediately (fast path)
|
||
if (result.matchedSupportedLocale && lowestDistance === 0) {
|
||
return result;
|
||
}
|
||
// === TIER 3: SLOW PATH - Full UTS #35 Distance Calculation ===
|
||
// Always run Tier 3 for full CLDR accuracy
|
||
// Tier 3 may find better matches than Tier 2's fallback approach
|
||
// findMatchingDistance is memoized, so repeated calculations are cached
|
||
requestedLocales.forEach((desired, i) => {
|
||
if (!result.distances[desired]) {
|
||
result.distances[desired] = {};
|
||
}
|
||
canonicalizedSupportedLocales.forEach((canonicalLocale, supportedIndex) => {
|
||
const originalSupported = supportedLocales[supportedIndex];
|
||
// findMatchingDistance is memoized via fast-memoize
|
||
// Use the canonical locale for distance calculation
|
||
const distance = findMatchingDistance(desired, canonicalLocale);
|
||
// Add some weight to the distance based on the order of the supported locales
|
||
// Add penalty for the order of the requested locales, which currently is 0 since ECMA-402
|
||
// doesn't really have room for weighted locales like `en; q=0.1`
|
||
const finalDistance = distance + 0 + i * 40;
|
||
// Store and return the original locale, not the canonical one
|
||
// Tier 3 overwrites Tier 2 distances (Tier 3 is more accurate)
|
||
result.distances[desired][originalSupported] = finalDistance;
|
||
if (finalDistance < lowestDistance) {
|
||
lowestDistance = finalDistance;
|
||
result.matchedDesiredLocale = desired;
|
||
result.matchedSupportedLocale = originalSupported;
|
||
}
|
||
});
|
||
});
|
||
if (lowestDistance >= threshold) {
|
||
result.matchedDesiredLocale = undefined;
|
||
result.matchedSupportedLocale = undefined;
|
||
}
|
||
return result;
|
||
}
|