my-project-public

repository

loading code, commits, and activity

repositories

loading repo index

#1	import { eld } from '@yutengjing/eld';
#2	import { existsSync } from 'node:fs';
#3
#4	import { readJSONSync, writeJSON } from '../utils/file';
#5	import { Logger } from '../utils/logger';
#6
#7	// 语言检测器初始化Promise，确保只初始化一次
#8	let initPromise: Promise<void> \| null = null;
#9
#10	/**
#11	* 初始化 ELD 语言检测器
#12	* 使用中等规模的 ngram 数据集
#13	*/
#14	async function initializeELD(): Promise<void> {
#15	if (!initPromise) {
#16	initPromise = (async () => {
#17	Logger.info('🔧 初始化 ELD 语言检测器...');
#18	await eld.init('M'); // 使用中等规模的数据集
#19	Logger.success('✅ ELD 语言检测器初始化完成');
#20	})();
#21	}
#22	return initPromise;
#23	}
#24
#25	// ISO 639-1 to project locale code mapping
#26	const languageMap: { [key: string]: string } = {
#27	en: 'en-US',
#28	zh: 'zh-CN',
#29	es: 'es-ES',
#30	fr: 'fr-FR',
#31	de: 'de-DE',
#32	it: 'it-IT',
#33	pt: 'pt-BR',
#34	ru: 'ru-RU',
#35	ja: 'ja-JP',
#36	ko: 'ko-KR',
#37	nl: 'nl-NL',
#38	pl: 'pl-PL',
#39	tr: 'tr-TR',
#40	vi: 'vi-VN',
#41	ar: 'ar',
#42	th: 'th-TH',
#43	bg: 'bg-BG',
#44	uk: 'uk-UA',
#45	el: 'el-GR',
#46	he: 'he-IL',
#47	hi: 'hi-IN',
#48	hu: 'hu-HU',
#49	cs: 'cs-CZ',
#50	fi: 'fi-FI',
#51	da: 'da-DK',
#52	nb: 'nb-NO',
#53	sv: 'sv-SE',
#54	ro: 'ro-RO',
#55	hr: 'hr-HR',
#56	sk: 'sk-SK',
#57	sl: 'sl-SI',
#58	lt: 'lt-LT',
#59	lv: 'lv-LV',
#60	et: 'et-EE',
#61	ca: 'ca-ES',
#62	gl: 'gl-ES',
#63	eu: 'eu-ES',
#64	is: 'is-IS',
#65	mt: 'mt-MT',
#66	ga: 'ga-IE',
#67	cy: 'cy-GB',
#68	gd: 'gd-GB',
#69	br: 'br-FR',
#70	kw: 'kw-GB',
#71	fa: 'fa-IR',
#72	ur: 'ur-PK',
#73	bn: 'bn-BD',
#74	ta: 'ta-IN',
#75	te: 'te-IN',
#76	kn: 'kn-IN',
#77	ml: 'ml-IN',
#78	gu: 'gu-IN',
#79	pa: 'pa-IN',
#80	or: 'or-IN',
#81	as: 'as-IN',
#82	ne: 'ne-NP',
#83	si: 'si-LK',
#84	my: 'my-MM',
#85	km: 'km-KH',
#86	lo: 'lo-LA',
#87	ka: 'ka-GE',
#88	hy: 'hy-AM',
#89	az: 'az-AZ',
#90	kk: 'kk-KZ',
#91	ky: 'ky-KG',
#92	tg: 'tg-TJ',
#93	uz: 'uz-UZ',
#94	mn: 'mn-MN',
#95	bo: 'bo-CN',
#96	ug: 'ug-CN',
#97	};
#98
#99	// 反向映射
#100	const reverseLanguageMap: { [key: string]: string } = Object.fromEntries(
#101	Object.entries(languageMap).map(([iso, locale]) => [locale, iso]),
#102	);
#103
#104	// 手动添加特殊情况的映射
#105	reverseLanguageMap['zh-TW'] = 'zh'; // 繁体中文也映射到 zh
#106
#107	// 置信度阈值 - 只标记高置信度的语言不匹配问题
#108	const MIN_CONFIDENCE_THRESHOLD = 0.8;
#109
#110	// i18n ignore 文件路径
#111	const I18N_IGNORE_FILE = '.i18nignore';
#112
#113	// 缓存的 ignore 列表
#114	let ignoreList: Set<string> \| null = null;
#115
#116	export interface LanguageValidationResult {
#117	confidence: number;
#118	detectedLanguage?: string;
#119	expectedLanguage: string;
#120	filePath: string;
#121	fixable?: boolean;
#122	issues?: FieldValidationIssue[];
#123	valid: boolean;
#124	}
#125
#126	export interface FieldValidationIssue {
#127	confidence: number;
#128	content: string;
#129	detectedLanguage: string;
#130	expectedLanguage: string;
#131	field: string;
#132	}
#133
#134	export interface ValidationStats {
#135	failed: number;
#136	fixed: number;
#137	ignored: number;
#138	lowConfidence: number;
#139	passed: number;
#140	total: number;
#141	}
#142
#143	/**
#144	* 读取 .i18nignore 文件
#145	* @returns ignore 文件列表
#146	*/
#147	function loadIgnoreList(): Set<string> {
#148	if (ignoreList !== null) {
#149	return ignoreList;
#150	}
#151
#152	ignoreList = new Set<string>();
#153
#154	if (existsSync(I18N_IGNORE_FILE)) {
#155	try {
#156	const content = require('node:fs').readFileSync(I18N_IGNORE_FILE, 'utf8');
#157	const lines = content
#158	.split('\n')
#159	.map((line) => line.trim())
#160	.filter((line) => line && !line.startsWith('#'));
#161
#162	lines.forEach((line) => ignoreList!.add(line));
#163	Logger.info(`已加载 ${ignoreList.size} 个忽略规则`);
#164	} catch (error) {
#165	Logger.warn(`读取 ${I18N_IGNORE_FILE} 失败: ${error}`);
#166	}
#167	}
#168
#169	return ignoreList;
#170	}
#171
#172	/**
#173	* 将文件路径添加到 ignore 列表
#174	* @param filePath - 文件路径
#175	*/
#176	function addToIgnoreList(filePath: string): void {
#177	const ignoreSet = loadIgnoreList();
#178	const relativePath = filePath.replace(process.cwd() + '/', '');
#179
#180	if (!ignoreSet.has(relativePath)) {
#181	ignoreSet.add(relativePath);
#182
#183	// 写入文件
#184	const ignoreArray = Array.from(ignoreSet).sort();
#185	const content =
#186	['# 语言验证忽略文件', '# 此文件中列出的路径将跳过语言验证', '', ...ignoreArray].join('\n') +
#187	'\n';
#188
#189	try {
#190	require('node:fs').writeFileSync(I18N_IGNORE_FILE, content, 'utf8');
#191	} catch (error) {
#192	Logger.error(`写入 ${I18N_IGNORE_FILE} 失败: ${error}`);
#193	}
#194	}
#195	}
#196
#197	/**
#198	* 检查文件是否在忽略列表中
#199	* @param filePath - 文件路径
#200	* @returns 是否应该忽略
#201	*/
#202	function isIgnored(filePath: string): boolean {
#203	const ignoreSet = loadIgnoreList();
#204	const relativePath = filePath.replace(process.cwd() + '/', '');
#205	return ignoreSet.has(relativePath);
#206	}
#207
#208	/**
#209	* 确保 ELD 已初始化（公开函数，供外部调用）
#210	*/
#211	export async function ensureELDInitialized(): Promise<void> {
#212	await initializeELD();
#213	}
#214
#215	/**
#216	* 检测文本的语言
#217	* @param text - 待检测的文本
#218	* @returns 语言检测结果对象
#219	*/
#220	async function detectLanguage(text: string): Promise<{
#221	confidence: number;
#222	detected: string;
#223	isReliable: boolean;
#224	scores: Record<string, number>;
#225	}> {
#226	if (!text?.trim()) {
#227	return {
#228	confidence: 0,
#229	detected: '',
#230	isReliable: false,
#231	scores: {},
#232	};
#233	}
#234
#235	// 确保 ELD 已初始化（这里不会重复初始化）
#236	await initializeELD();
#237
#238	const result = eld.detect(text);
#239	const scores = result.getScores();
#240	const topScore = Math.max(...Object.values(scores));
#241
#242	return {
#243	confidence: topScore,
#244	detected: result.language,
#245	isReliable: result.isReliable(),
#246	scores: scores,
#247	};
#248	}
#249
#250	/**
#251	* 从翻译数据中提取字段级文本内容
#252	* @param data - 翻译数据对象
#253	* @returns 字段路径到文本内容的映射
#254	*/
#255	function extractFieldTexts(data: any): Map<string, string> {
#256	const fieldTexts = new Map<string, string>();
#257
#258	function traverse(obj: any, path: string = '') {
#259	if (typeof obj === 'string' && obj.trim().length > 10) {
#260	fieldTexts.set(path, obj);
#261	} else if (Array.isArray(obj)) {
#262	// 特殊处理：examples 和 openingQuestions 字段整体检查
#263	if (path === 'examples' \|\| path.endsWith('.examples')) {
#264	const combinedContent = obj
#265	.map((item) => {
#266	if (typeof item === 'string') return item;
#267	if (item && typeof item === 'object') {
#268	// 提取示例中的内容
#269	const contents: string[] = [];
#270	if (item.content) contents.push(item.content);
#271	if (item.role) contents.push(item.role);
#272	return contents.join(' ');
#273	}
#274	return '';
#275	})
#276	.filter((content) => content.trim().length > 0)
#277	.join(' ');
#278
#279	if (combinedContent.trim().length > 20) {
#280	// examples 需要更多文本才能准确检测
#281	fieldTexts.set(path, combinedContent);
#282	}
#283	}
#284	// 特殊处理：openingQuestions 字段整体检查
#285	else if (path === 'openingQuestions' \|\| path.endsWith('.openingQuestions')) {
#286	const combinedContent = obj
#287	.filter((item) => typeof item === 'string' && item.trim().length > 0)
#288	.join(' ');
#289
#290	if (combinedContent.trim().length > 20) {
#291	// openingQuestions 需要更多文本才能准确检测
#292	fieldTexts.set(path, combinedContent);
#293	}
#294	}
#295	// 排除 tags 字段：tags 通常是技术标签，不需要语言检测
#296	else if (path === 'tags' \|\| path === 'meta.tags' \|\| path.endsWith('.tags')) {
#297	// 跳过 tags 字段
#298	} else {
#299	// 其他数组字段逐个检查
#300	obj.forEach((item, index) => {
#301	if (typeof item === 'string' && item.trim().length > 10) {
#302	fieldTexts.set(`${path}[${index}]`, item);
#303	} else if (typeof item === 'object') {
#304	traverse(item, `${path}[${index}]`);
#305	}
#306	});
#307	}
#308	} else if (obj && typeof obj === 'object') {
#309	Object.entries(obj).forEach(([key, value]) => {
#310	const currentPath = path ? `${path}.${key}` : key;
#311	traverse(value, currentPath);
#312	});
#313	}
#314	}
#315
#316	traverse(data);
#317	return fieldTexts;
#318	}
#319
#320	/**
#321	* 从翻译数据中提取可检测的文本内容
#322	* @param data - 翻译数据对象
#323	* @returns 可检测的文本数组
#324	*/
#325	function extractDetectableText(data: any): string[] {
#326	const texts: string[] = [];
#327
#328	function traverse(obj: any) {
#329	if (typeof obj === 'string' && obj.trim().length > 10) {
#330	texts.push(obj);
#331	} else if (Array.isArray(obj)) {
#332	obj.forEach((item) => {
#333	if (typeof item === 'string' && item.trim().length > 10) {
#334	texts.push(item);
#335	} else if (typeof item === 'object') {
#336	traverse(item);
#337	}
#338	});
#339	} else if (obj && typeof obj === 'object') {
#340	Object.values(obj).forEach((value) => {
#341	traverse(value);
#342	});
#343	}
#344	}
#345
#346	traverse(data);
#347	return texts;
#348	}
#349
#350	/**
#351	* 获取期望的语言代码
#352	* @param locale - 本地化代码
#353	* @returns ISO 639-1 语言代码
#354	*/
#355	function getExpectedLanguage(locale: string): string {
#356	return reverseLanguageMap[locale] \|\| 'en';
#357	}
#358
#359	/**
#360	* 根据路径删除对象中的字段
#361	* @param obj - 目标对象
#362	* @param path - 字段路径 (如: "config.systemRole" 或 "examples[0].content")
#363	* @returns 是否成功删除
#364	*/
#365	// eslint-disable-next-line @typescript-eslint/no-unused-vars
#366	async function removeFieldByPath(obj: any, path: string): Promise<boolean> {
#367	const pathParts = path.split(/[.[\]]+/).filter(Boolean);
#368
#369	if (pathParts.length === 0) return false;
#370
#371	let current = obj;
#372
#373	// 导航到目标字段的父级
#374	for (let i = 0; i < pathParts.length - 1; i++) {
#375	const part = pathParts[i];
#376
#377	if (current && typeof current === 'object') {
#378	current = current[part];
#379	} else {
#380	return false;
#381	}
#382	}
#383
#384	// 删除目标字段
#385	const lastKey = pathParts.at(-1);
#386	if (current && typeof current === 'object' && lastKey in current) {
#387	if (Array.isArray(current)) {
#388	const index = parseInt(lastKey);
#389	if (!isNaN(index) && index >= 0 && index < current.length) {
#390	current.splice(index, 1);
#391	return true;
#392	}
#393	} else {
#394	delete current[lastKey];
#395	return true;
#396	}
#397	}
#398
#399	return false;
#400	}
#401
#402	/**
#403	* 验证字段级语言匹配
#404	* @param data - 翻译数据
#405	* @param expectedLanguage - 期望的语言
#406	* @returns 字段验证问题列表
#407	*/
#408	async function validateFieldLanguages(
#409	data: any,
#410	expectedLanguage: string,
#411	): Promise<FieldValidationIssue[]> {
#412	const issues: FieldValidationIssue[] = [];
#413	const fieldTexts = extractFieldTexts(data);
#414
#415	for (const [fieldPath, text] of fieldTexts) {
#416	const detection = await detectLanguage(text);
#417	const detectedLanguage = detection.detected;
#418	const confidence = detection.confidence;
#419
#420	// 如果检测到的语言与期望语言不匹配，且置信度较高
#421	if (
#422	detectedLanguage &&
#423	detectedLanguage !== expectedLanguage &&
#424	confidence > MIN_CONFIDENCE_THRESHOLD
#425	) {
#426	issues.push({
#427	field: fieldPath,
#428	detectedLanguage,
#429	confidence,
#430	expectedLanguage,
#431	content: text.slice(0, 100) + (text.length > 100 ? '...' : ''),
#432	});
#433	}
#434	}
#435
#436	return issues;
#437	}
#438
#439	/**
#440	* 验证单个翻译文件的语言
#441	* @param filePath - 文件路径
#442	* @returns 验证结果
#443	*/
#444	export async function validateTranslationLanguage(
#445	filePath: string,
#446	): Promise<LanguageValidationResult> {
#447	try {
#448	// 检查是否在忽略列表中
#449	if (isIgnored(filePath)) {
#450	return {
#451	filePath,
#452	expectedLanguage: 'ignored',
#453	valid: true,
#454	confidence: 1,
#455	detectedLanguage: 'ignored',
#456	};
#457	}
#458
#459	const data = readJSONSync(filePath);
#460	const locale =
#461	filePath.match(/\.([a-z]{2}-[A-Z]{2})\.json$/)?.[1] \|\|
#462	filePath.match(/\.([a-z]{2})\.json$/)?.[1] \|\|
#463	(filePath.endsWith('index.json') ? 'en-US' : undefined);
#464
#465	if (!locale) {
#466	return {
#467	filePath,
#468	expectedLanguage: 'unknown',
#469	valid: false,
#470	confidence: 0,
#471	};
#472	}
#473
#474	const expectedLanguage = getExpectedLanguage(locale);
#475	const texts = extractDetectableText(data);
#476
#477	if (texts.length === 0) {
#478	return {
#479	filePath,
#480	expectedLanguage,
#481	valid: true,
#482	confidence: 1,
#483	};
#484	}
#485
#486	// 检测整体语言
#487	const combinedText = texts.join(' ');
#488	const detection = await detectLanguage(combinedText);
#489	const detectedLanguage = detection.detected;
#490	const confidence = detection.confidence;
#491
#492	// 检测字段级问题
#493	const fieldIssues = await validateFieldLanguages(data, expectedLanguage);
#494
#495	const languageMatches = detectedLanguage === expectedLanguage;
#496	const hasFieldIssues = fieldIssues.length > 0;
#497
#498	// 如果没有检测到语言
#499	if (!detectedLanguage) {
#500	return {
#501	filePath,
#502	expectedLanguage,
#503	valid: false,
#504	confidence: 0,
#505	issues: hasFieldIssues ? fieldIssues : undefined,
#506	fixable: true, // 无法检测语言的文件都可以用兜底修复
#507	};
#508	}
#509
#510	// 语言匹配检查
#511	if (languageMatches) {
#512	// 即使语言匹配，如果置信度很低，也可以用兜底修复
#513	const isLowConfidence = confidence < 0.4;
#514
#515	return {
#516	filePath,
#517	expectedLanguage,
#518	valid: !isLowConfidence, // 低置信度视为验证失败
#519	confidence,
#520	detectedLanguage,
#521	issues: hasFieldIssues ? fieldIssues : undefined,
#522	fixable: hasFieldIssues \|\| isLowConfidence, // 字段问题或低置信度都可以修复
#523	};
#524	}
#525
#526	// 语言不匹配
#527	return {
#528	filePath,
#529	expectedLanguage,
#530	valid: false,
#531	confidence,
#532	detectedLanguage,
#533	issues: hasFieldIssues ? fieldIssues : undefined,
#534	fixable: true, // 所有语言不匹配的文件都可以用兜底修复
#535	};
#536	} catch (error) {
#537	Logger.error(`验证文件失败: ${filePath} - ${error}`);
#538	return {
#539	filePath,
#540	expectedLanguage: 'unknown',
#541	valid: false,
#542	confidence: 0,
#543	};
#544	}
#545	}
#546
#547	/**
#548	* 获取对应的 en-US 兜底文件路径
#549	* @param filePath - 当前文件路径
#550	* @returns en-US 文件路径
#551	*/
#552	function getEnUsFallbackPath(filePath: string): string {
#553	// locales/agent-name/index.locale.json -> locales/agent-name/index.json
#554	return filePath.replace(/\.([a-z]{2}(-[A-Z]{2})?)\.json$/, '.json');
#555	}
#556
#557	/**
#558	* 从对象中获取指定路径的值
#559	* @param obj - 目标对象
#560	* @param path - 字段路径
#561	* @returns 字段值
#562	*/
#563	function getFieldValue(obj: any, path: string): any {
#564	const pathParts = path.split(/[.[\]]+/).filter(Boolean);
#565	let current = obj;
#566
#567	for (const part of pathParts) {
#568	if (current && typeof current === 'object' && part in current) {
#569	current = current[part];
#570	} else {
#571	return undefined;
#572	}
#573	}
#574
#575	return current;
#576	}
#577
#578	/**
#579	* 使用 en-US 兜底替换整个文件
#580	* @param filePath - 文件路径
#581	* @returns 是否成功修复
#582	*/
#583	export async function fixLanguageWithFallback(filePath: string): Promise<boolean> {
#584	try {
#585	const enUsPath = getEnUsFallbackPath(filePath);
#586
#587	// 检查 en-US 兜底文件是否存在
#588	if (!existsSync(enUsPath)) {
#589	Logger.error(`兜底文件不存在: ${enUsPath}`);
#590	return false;
#591	}
#592
#593	const enUsData = readJSONSync(enUsPath);
#594
#595	// 用 en-US 数据替换当前文件
#596	await writeJSON(filePath, enUsData);
#597
#598	// 将修复后的文件添加到忽略列表
#599	addToIgnoreList(filePath);
#600
#601	Logger.success(` 使用 en-US 兜底修复: ${filePath}`);
#602	return true;
#603	} catch (error) {
#604	Logger.error(`兜底修复失败: ${filePath} - ${error}`);
#605	return false;
#606	}
#607	}
#608
#609	/**
#610	* 修复翻译文件中的语言问题
#611	* @param filePath - 文件路径
#612	* @param issues - 验证问题列表
#613	* @returns 是否成功修复
#614	*/
#615	export async function fixLanguageIssues(
#616	filePath: string,
#617	issues: FieldValidationIssue[],
#618	): Promise<boolean> {
#619	try {
#620	const data = readJSONSync(filePath);
#621	const enUsPath = getEnUsFallbackPath(filePath);
#622
#623	// 检查 en-US 兜底文件是否存在
#624	if (!existsSync(enUsPath)) {
#625	Logger.error(`兜底文件不存在: ${enUsPath}`);
#626	return false;
#627	}
#628
#629	const enUsData = readJSONSync(enUsPath);
#630	let modified = false;
#631
#632	for (const issue of issues) {
#633	// 从 en-US 文件中获取对应字段的值
#634	const fallbackValue = getFieldValue(enUsData, issue.field);
#635
#636	if (fallbackValue !== undefined) {
#637	// 使用 en-US 的值替换问题字段
#638	const pathParts = issue.field.split(/[.[\]]+/).filter(Boolean);
#639	let current = data;
#640
#641	// 导航到父级对象
#642	for (let i = 0; i < pathParts.length - 1; i++) {
#643	const part = pathParts[i];
#644	if (current && typeof current === 'object') {
#645	if (!(part in current)) {
#646	current[part] = {};
#647	}
#648	current = current[part];
#649	}
#650	}
#651
#652	// 设置字段值
#653	const lastKey = pathParts.at(-1);
#654	if (current && typeof current === 'object') {
#655	current[lastKey] = fallbackValue;
#656	Logger.info(
#657	` 替换字段: ${issue.field} (检测为${issue.detectedLanguage}, 用 en-US 兜底)`,
#658	);
#659	modified = true;
#660	}
#661	} else {
#662	Logger.warn(` en-US 兜底文件中未找到字段: ${issue.field}`);
#663	}
#664	}
#665
#666	if (modified) {
#667	await writeJSON(filePath, data);
#668
#669	// 将修复后的文件添加到忽略列表
#670	addToIgnoreList(filePath);
#671
#672	Logger.success(` 修复完成: ${filePath}`);
#673	return true;
#674	}
#675
#676	return false;
#677	} catch (error) {
#678	Logger.error(`修复失败: ${filePath} - ${error}`);
#679	return false;
#680	}
#681	}
#682
#683	/**
#684	* 获取支持的语言列表 (ELD 支持的 ISO 639-1 语言代码)
#685	* @returns 支持的语言代码数组
#686	*/
#687	export function getSupportedLanguages(): string[] {
#688	// ELD 支持的主要语言 (ISO 639-1 代码)
#689	return [
#690	'af',
#691	'ar',
#692	'az',
#693	'be',
#694	'bg',
#695	'bn',
#696	'bs',
#697	'ca',
#698	'cs',
#699	'cy',
#700	'da',
#701	'de',
#702	'el',
#703	'en',
#704	'eo',
#705	'es',
#706	'et',
#707	'eu',
#708	'fa',
#709	'fi',
#710	'fr',
#711	'ga',
#712	'gl',
#713	'gu',
#714	'he',
#715	'hi',
#716	'hr',
#717	'ht',
#718	'hu',
#719	'hy',
#720	'id',
#721	'is',
#722	'it',
#723	'ja',
#724	'ka',
#725	'kk',
#726	'km',
#727	'kn',
#728	'ko',
#729	'ku',
#730	'ky',
#731	'la',
#732	'lb',
#733	'lo',
#734	'lt',
#735	'lv',
#736	'mk',
#737	'ml',
#738	'mn',
#739	'mr',
#740	'ms',
#741	'mt',
#742	'my',
#743	'ne',
#744	'nl',
#745	'no',
#746	'pa',
#747	'pl',
#748	'pt',
#749	'ro',
#750	'ru',
#751	'si',
#752	'sk',
#753	'sl',
#754	'so',
#755	'sq',
#756	'sr',
#757	'sv',
#758	'sw',
#759	'ta',
#760	'te',
#761	'th',
#762	'tl',
#763	'tr',
#764	'uk',
#765	'ur',
#766	'uz',
#767	'vi',
#768	'yi',
#769	'zh',
#770	];
#771	}
#772
#773	/**
#774	* 检查语言是否受支持
#775	* @param langCode - 语言代码
#776	* @returns 是否支持该语言
#777	*/
#778	export function isLanguageSupported(langCode: string): boolean {
#779	return getSupportedLanguages().includes(langCode);
#780	}
#781
#782
#783

z6Mkq5mY3JWtxoxUobWcfNHm7AkRubgSWEZTkBVqZXJviFZ5/my-project-public