mirror of
https://github.com/alam00000/bentopdf.git
synced 2026-11-07 16:41:00 +01:00
feat: translate all pages, UI messages and docs and fix SEO redirects and indexing
This commit is contained in:
alam00000
@@ -0,0 +1,36 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { isIndexingBlocked, resolveSiteUrl } from './indexing-policy.mjs';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const DIST_DIR = path.resolve(__dirname, '../dist');
|
||||
const NOINDEX_META = '<meta name="robots" content="noindex, follow" />';
|
||||
|
||||
function listHtml(dir) {
|
||||
return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) return listHtml(full);
|
||||
return entry.name.endsWith('.html') ? [full] : [];
|
||||
});
|
||||
}
|
||||
|
||||
if (!isIndexingBlocked()) {
|
||||
console.log('Indexing policy: search engines may index this build.');
|
||||
} else {
|
||||
let updated = 0;
|
||||
for (const file of listHtml(DIST_DIR)) {
|
||||
const html = fs.readFileSync(file, 'utf-8');
|
||||
const robots = /<meta\s+name=["']robots["'][^>]*>/i;
|
||||
const next = robots.test(html)
|
||||
? html.replace(robots, NOINDEX_META)
|
||||
: html.replace(/<\/head>/i, ` ${NOINDEX_META}\n </head>`);
|
||||
if (next !== html) {
|
||||
fs.writeFileSync(file, next);
|
||||
updated++;
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`Indexing policy: SITE_URL is ${resolveSiteUrl()}, so ${updated} pages were marked noindex. Set ALLOW_INDEXING=true to let search engines index this deployment.`
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,571 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { serializeJsonLd } from './json-ld.mjs';
|
||||
|
||||
const INLINE_TAGS = new Set([
|
||||
'a',
|
||||
'abbr',
|
||||
'b',
|
||||
'bdi',
|
||||
'bdo',
|
||||
'cite',
|
||||
'data',
|
||||
'del',
|
||||
'dfn',
|
||||
'em',
|
||||
'i',
|
||||
'ins',
|
||||
'mark',
|
||||
'q',
|
||||
's',
|
||||
'small',
|
||||
'span',
|
||||
'strong',
|
||||
'sub',
|
||||
'sup',
|
||||
'time',
|
||||
'u',
|
||||
]);
|
||||
|
||||
const ALWAYS_OPAQUE_TAGS = new Set([
|
||||
'audio',
|
||||
'br',
|
||||
'canvas',
|
||||
'code',
|
||||
'iframe',
|
||||
'img',
|
||||
'input',
|
||||
'kbd',
|
||||
'math',
|
||||
'samp',
|
||||
'svg',
|
||||
'var',
|
||||
'video',
|
||||
'wbr',
|
||||
]);
|
||||
|
||||
const SKIPPED_TAGS = new Set([
|
||||
'head',
|
||||
'noscript',
|
||||
'pre',
|
||||
'script',
|
||||
'style',
|
||||
'template',
|
||||
'textarea',
|
||||
]);
|
||||
|
||||
const TRANSLATABLE_ATTRIBUTES = ['alt', 'title', 'placeholder', 'aria-label'];
|
||||
|
||||
const META_SELECTORS = [
|
||||
'meta[name="description"]',
|
||||
'meta[property="og:title"]',
|
||||
'meta[property="og:description"]',
|
||||
'meta[name="twitter:title"]',
|
||||
'meta[name="twitter:description"]',
|
||||
];
|
||||
|
||||
const PROTECTED_TERMS = [
|
||||
'BentoPDF',
|
||||
'Hyper Compress',
|
||||
'Kura',
|
||||
'GitHub',
|
||||
'PyMuPDF',
|
||||
'Ghostscript',
|
||||
'CoherentPDF',
|
||||
'LibreOffice',
|
||||
'Tesseract',
|
||||
'DigitalOcean',
|
||||
'Fubarpie',
|
||||
'Trustpilot',
|
||||
'LinkedIn',
|
||||
'Discord',
|
||||
'WebAssembly',
|
||||
'PDF/A',
|
||||
'PDF/UA',
|
||||
'PDF/X',
|
||||
'PDF',
|
||||
];
|
||||
|
||||
const PROTECTED_PATTERN = new RegExp(
|
||||
PROTECTED_TERMS.map((term) =>
|
||||
term.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')
|
||||
).join('|'),
|
||||
'gi'
|
||||
);
|
||||
|
||||
const LETTERS = /\p{L}{2}/u;
|
||||
const PLACEHOLDER_SOURCE = '<(\\/?)(\\d+)(\\/?)>';
|
||||
const SPACES = /[ \t\r\n\f]+/g;
|
||||
|
||||
function placeholders(text) {
|
||||
return Array.from(text.matchAll(new RegExp(PLACEHOLDER_SOURCE, 'g')));
|
||||
}
|
||||
|
||||
function hasPlaceholder(text) {
|
||||
return new RegExp(PLACEHOLDER_SOURCE).test(text);
|
||||
}
|
||||
|
||||
export function segmentId(markup) {
|
||||
return createHash('sha1').update(markup).digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
function escapeText(text) {
|
||||
return text
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
function decodeText(text) {
|
||||
return text
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
function isTranslatable(markup) {
|
||||
const text = markup
|
||||
.replace(new RegExp(PLACEHOLDER_SOURCE, 'g'), ' ')
|
||||
.replace(PROTECTED_PATTERN, ' ');
|
||||
return LETTERS.test(text);
|
||||
}
|
||||
|
||||
function normalize(text) {
|
||||
return text.replace(SPACES, ' ').trim();
|
||||
}
|
||||
|
||||
function tagOf(element) {
|
||||
return element.tagName.toLowerCase();
|
||||
}
|
||||
|
||||
function isInlineLike(element) {
|
||||
const tag = tagOf(element);
|
||||
if (ALWAYS_OPAQUE_TAGS.has(tag)) return true;
|
||||
if (!INLINE_TAGS.has(tag)) return false;
|
||||
if (element.hasAttribute('data-i18n')) return true;
|
||||
return Array.from(element.children).every(isInlineLike);
|
||||
}
|
||||
|
||||
function isSkippedBlock(element) {
|
||||
return (
|
||||
SKIPPED_TAGS.has(tagOf(element)) ||
|
||||
element.hasAttribute('data-i18n') ||
|
||||
element.getAttribute('translate') === 'no' ||
|
||||
element.hasAttribute('data-no-i18n') ||
|
||||
element.hasAttribute('contenteditable')
|
||||
);
|
||||
}
|
||||
|
||||
function isOpaque(element) {
|
||||
return (
|
||||
ALWAYS_OPAQUE_TAGS.has(tagOf(element)) ||
|
||||
element.hasAttribute('data-i18n') ||
|
||||
element.getAttribute('translate') === 'no' ||
|
||||
!LETTERS.test(element.textContent || '')
|
||||
);
|
||||
}
|
||||
|
||||
function serialize(nodes, state) {
|
||||
let out = '';
|
||||
for (const node of nodes) {
|
||||
if (node.nodeType === 3) {
|
||||
out += escapeText(node.data);
|
||||
} else if (node.nodeType === 1) {
|
||||
state.count += 1;
|
||||
const index = state.count;
|
||||
state.elements[index] = node;
|
||||
if (isOpaque(node)) {
|
||||
out += `<${index}/>`;
|
||||
} else {
|
||||
out += `<${index}>${serialize(Array.from(node.childNodes), state)}</${index}>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function parseTranslation(text, elements) {
|
||||
const root = { children: [] };
|
||||
const stack = [root];
|
||||
const seen = new Set();
|
||||
let last = 0;
|
||||
const pushText = (piece) => {
|
||||
if (!piece) return true;
|
||||
if (/[<>]/.test(piece)) return false;
|
||||
stack[stack.length - 1].children.push({ text: decodeText(piece) });
|
||||
return true;
|
||||
};
|
||||
for (const match of placeholders(text)) {
|
||||
if (!pushText(text.slice(last, match.index))) return null;
|
||||
last = match.index + match[0].length;
|
||||
const closing = match[1] === '/';
|
||||
const selfClosing = match[3] === '/';
|
||||
const index = Number(match[2]);
|
||||
const source = elements[index];
|
||||
if (!source || (closing && selfClosing)) return null;
|
||||
if (selfClosing) {
|
||||
if (!isOpaque(source) || seen.has(index)) return null;
|
||||
seen.add(index);
|
||||
stack[stack.length - 1].children.push({ element: index });
|
||||
} else if (!closing) {
|
||||
if (isOpaque(source) || seen.has(index)) return null;
|
||||
seen.add(index);
|
||||
const node = { element: index, children: [] };
|
||||
stack[stack.length - 1].children.push(node);
|
||||
stack.push(node);
|
||||
} else {
|
||||
const top = stack[stack.length - 1];
|
||||
if (top.element !== index) return null;
|
||||
stack.pop();
|
||||
}
|
||||
}
|
||||
if (!pushText(text.slice(last))) return null;
|
||||
if (stack.length !== 1) return null;
|
||||
if (seen.size !== Object.keys(elements).length) return null;
|
||||
return root.children;
|
||||
}
|
||||
|
||||
function buildFragment(document, children, elements) {
|
||||
const fragment = document.createDocumentFragment();
|
||||
for (const child of children) {
|
||||
if ('text' in child) {
|
||||
fragment.appendChild(document.createTextNode(child.text));
|
||||
} else if (child.children) {
|
||||
const clone = elements[child.element].cloneNode(false);
|
||||
clone.appendChild(buildFragment(document, child.children, elements));
|
||||
fragment.appendChild(clone);
|
||||
} else {
|
||||
fragment.appendChild(elements[child.element].cloneNode(true));
|
||||
}
|
||||
}
|
||||
return fragment;
|
||||
}
|
||||
|
||||
function significantNodes(nodes) {
|
||||
return nodes.filter(
|
||||
(node) => node.nodeType !== 3 || normalize(node.data) !== ''
|
||||
);
|
||||
}
|
||||
|
||||
function handleSegment(markup, tag, context, apply) {
|
||||
const { options, stats } = context;
|
||||
const id = segmentId(markup);
|
||||
stats.found += 1;
|
||||
if (options.record) options.record(id, markup, tag);
|
||||
if (!options.translate) return;
|
||||
const translation = options.translate(id);
|
||||
if (!translation) return;
|
||||
if (apply(translation)) stats.translated += 1;
|
||||
else stats.invalid.add(id);
|
||||
}
|
||||
|
||||
function handleAttributes(element, context) {
|
||||
for (const name of TRANSLATABLE_ATTRIBUTES) {
|
||||
if (!element.hasAttribute(name)) continue;
|
||||
if (element.hasAttribute(`data-i18n-${name}`)) continue;
|
||||
const value = normalize(element.getAttribute(name));
|
||||
if (!isTranslatable(escapeText(value))) continue;
|
||||
handleSegment(escapeText(value), tagOf(element), context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
element.setAttribute(name, decodeText(translation));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function handleRun(nodes, tag, context) {
|
||||
const significant = significantNodes(nodes);
|
||||
if (significant.length === 0) return;
|
||||
for (const node of nodes) {
|
||||
if (node.nodeType !== 1) continue;
|
||||
handleAttributes(node, context);
|
||||
node
|
||||
.querySelectorAll('*')
|
||||
.forEach((child) => handleAttributes(child, context));
|
||||
}
|
||||
if (
|
||||
significant.length === 1 &&
|
||||
significant[0].nodeType === 1 &&
|
||||
!isOpaque(significant[0])
|
||||
) {
|
||||
handleRun(Array.from(significant[0].childNodes), tag, context);
|
||||
return;
|
||||
}
|
||||
const state = { count: 0, elements: {} };
|
||||
const markup = normalize(serialize(nodes, state));
|
||||
if (!isTranslatable(markup)) return;
|
||||
handleSegment(markup, tag, context, (translation) => {
|
||||
const children = parseTranslation(translation, state.elements);
|
||||
if (!children) return false;
|
||||
const document = significant[0].ownerDocument;
|
||||
const fragment = buildFragment(document, children, state.elements);
|
||||
const anchor = nodes[0];
|
||||
anchor.parentNode.insertBefore(fragment, anchor);
|
||||
nodes.forEach((node) => node.parentNode.removeChild(node));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function visitChildren(parent, context) {
|
||||
const tag = tagOf(parent);
|
||||
let run = [];
|
||||
const flush = () => {
|
||||
if (run.length > 0) handleRun(run, tag, context);
|
||||
run = [];
|
||||
};
|
||||
for (const node of Array.from(parent.childNodes)) {
|
||||
if (node.nodeType === 3) {
|
||||
run.push(node);
|
||||
} else if (node.nodeType === 1 && isInlineLike(node)) {
|
||||
run.push(node);
|
||||
} else {
|
||||
flush();
|
||||
if (node.nodeType === 1 && !isSkippedBlock(node)) {
|
||||
handleAttributes(node, context);
|
||||
visitChildren(node, context);
|
||||
}
|
||||
}
|
||||
}
|
||||
flush();
|
||||
}
|
||||
|
||||
function processMeta(document, context) {
|
||||
const title = document.querySelector('title');
|
||||
if (title) {
|
||||
const value = normalize(title.textContent || '');
|
||||
if (isTranslatable(escapeText(value))) {
|
||||
handleSegment(escapeText(value), 'title', context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
title.textContent = decodeText(translation);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const selector of META_SELECTORS) {
|
||||
document.querySelectorAll(selector).forEach((meta) => {
|
||||
const value = normalize(meta.getAttribute('content') || '');
|
||||
if (!isTranslatable(escapeText(value))) return;
|
||||
handleSegment(escapeText(value), 'meta', context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
meta.setAttribute('content', decodeText(translation));
|
||||
return true;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const STRUCTURED_DATA_TEXT_KEYS = new Set([
|
||||
'name',
|
||||
'description',
|
||||
'text',
|
||||
'headline',
|
||||
]);
|
||||
|
||||
const STRUCTURED_DATA_SKIPPED_TYPES = new Set([
|
||||
'BreadcrumbList',
|
||||
'FAQPage',
|
||||
'ImageObject',
|
||||
'ListItem',
|
||||
'Offer',
|
||||
'Organization',
|
||||
'Person',
|
||||
'Question',
|
||||
'WebSite',
|
||||
]);
|
||||
|
||||
function walkStructuredData(value, handle) {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item) => walkStructuredData(item, handle));
|
||||
return;
|
||||
}
|
||||
if (!value || typeof value !== 'object') return;
|
||||
if (STRUCTURED_DATA_SKIPPED_TYPES.has(value['@type'])) return;
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (typeof child === 'string') {
|
||||
if (STRUCTURED_DATA_TEXT_KEYS.has(key)) handle(value, key);
|
||||
} else if (key === 'featureList' && Array.isArray(child)) {
|
||||
child.forEach((item, index) => {
|
||||
if (typeof item === 'string') handle(child, index);
|
||||
});
|
||||
} else {
|
||||
walkStructuredData(child, handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function processStructuredData(document, context) {
|
||||
document
|
||||
.querySelectorAll('script[type="application/ld+json"]')
|
||||
.forEach((script) => {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(script.textContent || '');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
let changed = false;
|
||||
walkStructuredData(data, (container, key) => {
|
||||
const value = normalize(container[key]);
|
||||
if (!isTranslatable(escapeText(value))) return;
|
||||
handleSegment(escapeText(value), 'ld', context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
container[key] = decodeText(translation);
|
||||
changed = true;
|
||||
return true;
|
||||
});
|
||||
});
|
||||
if (changed) script.textContent = serializeJsonLd(data);
|
||||
});
|
||||
}
|
||||
|
||||
const BLOCK_TAGS = new Set([
|
||||
'p',
|
||||
'div',
|
||||
'ul',
|
||||
'ol',
|
||||
'li',
|
||||
'table',
|
||||
'tr',
|
||||
'td',
|
||||
'th',
|
||||
'h1',
|
||||
'h2',
|
||||
'h3',
|
||||
'h4',
|
||||
'h5',
|
||||
'h6',
|
||||
'blockquote',
|
||||
'section',
|
||||
]);
|
||||
|
||||
function blockText(node) {
|
||||
if (node.nodeType !== 1) return node.textContent || '';
|
||||
const text = Array.from(node.childNodes).map(blockText).join('');
|
||||
return BLOCK_TAGS.has(tagOf(node)) ? ` ${text} ` : text;
|
||||
}
|
||||
|
||||
function rebuildFaqSchema(document) {
|
||||
const items = [];
|
||||
for (const details of document.querySelectorAll('details.faq-d')) {
|
||||
const summary = details.querySelector('summary');
|
||||
if (!summary) continue;
|
||||
const question = normalize(summary.textContent || '');
|
||||
const answerText = normalize(
|
||||
Array.from(details.childNodes)
|
||||
.filter((child) => child !== summary)
|
||||
.map(blockText)
|
||||
.join(' ')
|
||||
);
|
||||
if (!question || !answerText) continue;
|
||||
items.push({
|
||||
'@type': 'Question',
|
||||
name: question,
|
||||
acceptedAnswer: { '@type': 'Answer', text: answerText },
|
||||
});
|
||||
}
|
||||
if (items.length === 0) return;
|
||||
document
|
||||
.querySelectorAll('script[type="application/ld+json"]')
|
||||
.forEach((script) => {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(script.textContent || '');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (data && data['@type'] === 'FAQPage') {
|
||||
script.textContent = serializeJsonLd({ ...data, mainEntity: items });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function processDocument(document, options) {
|
||||
const context = {
|
||||
options,
|
||||
stats: { found: 0, translated: 0, invalid: new Set() },
|
||||
};
|
||||
if (!options.skipMeta) processMeta(document, context);
|
||||
const roots = options.roots
|
||||
? options.roots(document)
|
||||
: [document.body].filter(Boolean);
|
||||
for (const root of roots) {
|
||||
if (!isSkippedBlock(root)) {
|
||||
handleAttributes(root, context);
|
||||
visitChildren(root, context);
|
||||
}
|
||||
}
|
||||
if (!options.skipMeta) {
|
||||
if (options.translate) rebuildFaqSchema(document);
|
||||
processStructuredData(document, context);
|
||||
}
|
||||
return {
|
||||
found: context.stats.found,
|
||||
translated: context.stats.translated,
|
||||
invalid: [...context.stats.invalid],
|
||||
};
|
||||
}
|
||||
|
||||
export const ENGLISH_ONLY_PAGES = new Set(['terms', 'privacy', 'licensing']);
|
||||
|
||||
export function processPage(document, pageName, options) {
|
||||
if (ENGLISH_ONLY_PAGES.has(pageName)) {
|
||||
return processDocument(document, {
|
||||
...options,
|
||||
skipMeta: true,
|
||||
roots: shellRoots,
|
||||
});
|
||||
}
|
||||
return processDocument(document, options);
|
||||
}
|
||||
|
||||
export function shellRoots(document) {
|
||||
return Array.from(document.querySelectorAll('header, nav, footer')).filter(
|
||||
(element) => !element.parentElement?.closest('header, nav, footer')
|
||||
);
|
||||
}
|
||||
|
||||
function placeholderParents(markup) {
|
||||
const stack = [];
|
||||
const parents = [];
|
||||
for (const match of placeholders(markup)) {
|
||||
const [, closing, index, selfClosing] = match;
|
||||
if (closing) {
|
||||
if (stack.pop() !== index) return null;
|
||||
} else {
|
||||
parents.push(`${index}:${stack.at(-1) ?? ''}`);
|
||||
if (!selfClosing) stack.push(index);
|
||||
}
|
||||
}
|
||||
return stack.length === 0 ? parents.sort().join(',') : null;
|
||||
}
|
||||
|
||||
function placeholderSignature(markup) {
|
||||
return placeholders(markup)
|
||||
.map((match) => `${match[1]}${match[2]}${match[3]}`)
|
||||
.sort()
|
||||
.join(',');
|
||||
}
|
||||
|
||||
export function validateTranslationPair(sourceMarkup, translation) {
|
||||
if (typeof translation !== 'string' || translation.trim() === '') {
|
||||
return 'empty';
|
||||
}
|
||||
if (
|
||||
placeholderSignature(sourceMarkup) !== placeholderSignature(translation)
|
||||
) {
|
||||
return 'placeholders differ';
|
||||
}
|
||||
const nesting = placeholderParents(translation);
|
||||
if (nesting === null || nesting !== placeholderParents(sourceMarkup)) {
|
||||
return 'placeholders nested differently';
|
||||
}
|
||||
const stripped = translation.replace(new RegExp(PLACEHOLDER_SOURCE, 'g'), '');
|
||||
if (/[<>]/.test(stripped)) return 'contains raw angle brackets';
|
||||
const unknownEntity = stripped.match(
|
||||
/&(?!(?:amp|lt|gt|quot|nbsp|#39);)[#\w]+;/
|
||||
);
|
||||
if (unknownEntity) return `unsupported entity ${unknownEntity[0]}`;
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
export function listDocsSourcePages(docsDir, languageCodes) {
|
||||
const codes = new Set(languageCodes);
|
||||
const pages = [];
|
||||
const walk = (dir, relative) => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (
|
||||
entry.name.startsWith('.') ||
|
||||
entry.name === 'node_modules' ||
|
||||
entry.name === 'public'
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const next = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
if (!relative && codes.has(entry.name)) continue;
|
||||
walk(path.join(dir, entry.name), next);
|
||||
} else if (entry.name.endsWith('.md')) {
|
||||
pages.push(next);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(docsDir, '');
|
||||
return pages.sort();
|
||||
}
|
||||
|
||||
export function completeDocsLanguages(docsDir, languageCodes, requested) {
|
||||
const pages = listDocsSourcePages(docsDir, languageCodes);
|
||||
const complete = [];
|
||||
const incomplete = [];
|
||||
for (const lang of languageCodes) {
|
||||
if (!fs.existsSync(path.join(docsDir, lang))) continue;
|
||||
const missing = pages.filter(
|
||||
(page) => !fs.existsSync(path.join(docsDir, lang, page))
|
||||
);
|
||||
if (missing.length === 0) complete.push(lang);
|
||||
else
|
||||
incomplete.push({ lang, missing: missing.length, total: pages.length });
|
||||
}
|
||||
const wanted =
|
||||
!requested || requested === 'all'
|
||||
? complete
|
||||
: requested
|
||||
.split(',')
|
||||
.map((lang) => lang.trim())
|
||||
.filter((lang) => complete.includes(lang));
|
||||
return { enabled: wanted, incomplete };
|
||||
}
|
||||
@@ -0,0 +1,592 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const DOCS_DIR = path.join(ROOT, 'docs');
|
||||
const LOCALES_DIR = path.join(ROOT, 'public/locales');
|
||||
const FILE_EXTENSION = /\.[a-z0-9]{2,5}$/i;
|
||||
|
||||
const SCRIPT_RANGES = {
|
||||
ar: /[\u0600-\u06ff]/,
|
||||
fa: /[\u0600-\u06ff]/,
|
||||
be: /[\u0400-\u04ff]/,
|
||||
ru: /[\u0400-\u04ff]/,
|
||||
uk: /[\u0400-\u04ff]/,
|
||||
ja: /[\u3040-\u30ff\u4e00-\u9fff]/,
|
||||
zh: /[\u4e00-\u9fff]/,
|
||||
'zh-TW': /[\u4e00-\u9fff]/,
|
||||
ko: /[\uac00-\ud7af]/,
|
||||
};
|
||||
|
||||
const ENGLISH_MARKERS = new Set([
|
||||
'the',
|
||||
'and',
|
||||
'your',
|
||||
'with',
|
||||
'you',
|
||||
'this',
|
||||
'that',
|
||||
'from',
|
||||
'are',
|
||||
'will',
|
||||
'how',
|
||||
'what',
|
||||
'when',
|
||||
'can',
|
||||
]);
|
||||
|
||||
function proseOf(markdown) {
|
||||
let fenced = false;
|
||||
return markdown
|
||||
.split('\n')
|
||||
.filter((line) => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
fenced = !fenced;
|
||||
return false;
|
||||
}
|
||||
return !fenced && !/^\s*(\||:::|---)/.test(line);
|
||||
})
|
||||
.join(' ')
|
||||
.replace(/`[^`]*`/g, ' ')
|
||||
.replace(/\]\([^)]*\)/g, ']')
|
||||
.replace(/https?:\/\/\S+/g, ' ');
|
||||
}
|
||||
|
||||
function languages() {
|
||||
return fs
|
||||
.readdirSync(LOCALES_DIR)
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry !== 'en' &&
|
||||
fs.statSync(path.join(LOCALES_DIR, entry)).isDirectory()
|
||||
)
|
||||
.sort();
|
||||
}
|
||||
|
||||
export function listSourceFiles() {
|
||||
const langs = new Set(languages());
|
||||
const files = [];
|
||||
const walk = (dir, relative) => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.name === 'node_modules' || entry.name.startsWith('.')) continue;
|
||||
if (entry.name === 'public') continue;
|
||||
const next = relative ? `${relative}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
if (!relative && langs.has(entry.name)) continue;
|
||||
walk(path.join(dir, entry.name), next);
|
||||
} else if (entry.name.endsWith('.md')) {
|
||||
files.push(next);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(DOCS_DIR, '');
|
||||
return files.sort();
|
||||
}
|
||||
|
||||
function wordCount(text) {
|
||||
return text.split(/\s+/).filter(Boolean).length;
|
||||
}
|
||||
|
||||
export function localizeLinks(markdown, lang) {
|
||||
const prefix = `/${lang}`;
|
||||
const shouldPrefix = (target) => {
|
||||
if (!target.startsWith('/') || target.startsWith('//')) return false;
|
||||
if (target === prefix || target.startsWith(`${prefix}/`)) return false;
|
||||
if (/^\/(images|docs)(\/|$)/.test(target)) return false;
|
||||
const pathOnly = target.split(/[?#]/)[0];
|
||||
return !FILE_EXTENSION.test(pathOnly);
|
||||
};
|
||||
const lines = markdown.split('\n');
|
||||
let fenced = false;
|
||||
return lines
|
||||
.map((line) => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
fenced = !fenced;
|
||||
return line;
|
||||
}
|
||||
if (fenced) return line;
|
||||
const linkRewritten = line.replace(
|
||||
/(?<!!)(\[[^\]]*\]\()(\/[^)\s]*)(\))/g,
|
||||
(match, open, target, close) =>
|
||||
shouldPrefix(target) ? `${open}${prefix}${target}${close}` : match
|
||||
);
|
||||
return linkRewritten.replace(
|
||||
/^(\s*(?:-\s+)?link:\s*)(\/\S*)\s*$/,
|
||||
(match, head, target) =>
|
||||
shouldPrefix(target) ? `${head}${prefix}${target}` : match
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
const SLUG_CONTROL = /[\u0000-\u001f]/g;
|
||||
const SLUG_SPECIAL =
|
||||
/[\s~`!@#$%^&*()\-_+=[\]{}|\\;:"'\u201c\u201d\u2018\u2019<>,.?/]+/g;
|
||||
const SLUG_COMBINING = /[\u0300-\u036f]/g;
|
||||
|
||||
function slugify(text) {
|
||||
return text
|
||||
.normalize('NFKD')
|
||||
.replace(SLUG_COMBINING, '')
|
||||
.replace(SLUG_CONTROL, '')
|
||||
.replace(SLUG_SPECIAL, '-')
|
||||
.replace(/-{2,}/g, '-')
|
||||
.replace(/^-+|-+$/g, '')
|
||||
.replace(/^(\d)/, '_$1')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function listHeadings(markdown) {
|
||||
const headings = [];
|
||||
const lines = markdown.split('\n');
|
||||
let fenced = false;
|
||||
let inFrontmatter = lines[0] === '---';
|
||||
lines.forEach((line, index) => {
|
||||
if (inFrontmatter) {
|
||||
if (index > 0 && line === '---') inFrontmatter = false;
|
||||
return;
|
||||
}
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
fenced = !fenced;
|
||||
return;
|
||||
}
|
||||
if (fenced) return;
|
||||
const match = line.match(/^(#{1,6})\s+(.*?)\s*$/);
|
||||
if (!match) return;
|
||||
const explicit = match[2].match(/\s*\{#([^}]+)\}\s*$/);
|
||||
const text = match[2].replace(/\s*\{#[^}]+\}\s*$/, '');
|
||||
headings.push({
|
||||
line: index,
|
||||
text,
|
||||
explicitId: explicit ? explicit[1] : null,
|
||||
});
|
||||
});
|
||||
return headings;
|
||||
}
|
||||
|
||||
function headingId(text) {
|
||||
return slugify(
|
||||
text
|
||||
.replace(/`/g, '')
|
||||
.replace(/!?\[([^\]]*)\]\([^)]*\)/g, '$1')
|
||||
.replace(/\*/g, '')
|
||||
);
|
||||
}
|
||||
|
||||
export function quoteFrontmatter(markdown) {
|
||||
const match = markdown.match(/^---\n([\s\S]*?)\n---\n/);
|
||||
if (!match) return markdown;
|
||||
const body = match[1]
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
const field = line.match(/^(\s*(?:- )?[\w-]+):[ \t]+(.+)$/);
|
||||
if (!field) return line;
|
||||
const value = field[2].trim();
|
||||
if (/^["'|>]/.test(value)) return line;
|
||||
if (!/: |\s#|^[&*!%@`{[]/.test(value)) return line;
|
||||
return `${field[1]}: "${value.replace(/\\/g, '\\\\').replace(/"/g, '\\"')}"`;
|
||||
})
|
||||
.join('\n');
|
||||
return `---\n${body}\n---\n${markdown.slice(match[0].length)}`;
|
||||
}
|
||||
|
||||
function headingIds(markdown) {
|
||||
const used = new Map();
|
||||
return listHeadings(markdown).map((heading) => {
|
||||
if (heading.explicitId) return heading.explicitId;
|
||||
const base = headingId(heading.text);
|
||||
const count = used.get(base) || 0;
|
||||
used.set(base, count + 1);
|
||||
return count === 0 ? base : `${base}-${count}`;
|
||||
});
|
||||
}
|
||||
|
||||
export function removeGeneratedAnchors(source, translated) {
|
||||
const english = listHeadings(source);
|
||||
const target = listHeadings(translated);
|
||||
if (english.length !== target.length) return translated;
|
||||
const lines = translated.split('\n');
|
||||
target.forEach((heading, index) => {
|
||||
if (english[index].explicitId) {
|
||||
lines[heading.line] = lines[heading.line].replace(
|
||||
/\s*\{#\\?[^}]+\}\s*$/,
|
||||
` {#${english[index].explicitId}}`
|
||||
);
|
||||
} else if (heading.explicitId) {
|
||||
lines[heading.line] = lines[heading.line].replace(
|
||||
/\s*\{#\\?[^}]+\}\s*$/,
|
||||
''
|
||||
);
|
||||
}
|
||||
});
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
export function anchorMap(source, translated) {
|
||||
const english = headingIds(source);
|
||||
const target = headingIds(translated);
|
||||
const map = new Map();
|
||||
if (english.length !== target.length) return { map, ids: new Set(target) };
|
||||
english.forEach((id, index) => map.set(id, target[index]));
|
||||
return { map, ids: new Set(target) };
|
||||
}
|
||||
|
||||
function linkedDocsFile(target, currentFile, lang) {
|
||||
const pathPart = target.split('#')[0].split('?')[0];
|
||||
let relative;
|
||||
if (pathPart === '') {
|
||||
return currentFile;
|
||||
} else if (pathPart === `/${lang}` || pathPart.startsWith(`/${lang}/`)) {
|
||||
relative = pathPart.slice(lang.length + 2);
|
||||
} else if (!pathPart.startsWith('/') && !/^[a-z]+:/i.test(pathPart)) {
|
||||
relative = path.posix.join(path.posix.dirname(currentFile), pathPart);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
relative = relative.replace(/\.(md|html)$/, '').replace(/^\/+/, '');
|
||||
if (relative === '' || relative.endsWith('/')) return `${relative}index.md`;
|
||||
return `${relative}.md`;
|
||||
}
|
||||
|
||||
export function translateAnchors(markdown, currentFile, lang, anchors) {
|
||||
let fenced = false;
|
||||
return markdown
|
||||
.split('\n')
|
||||
.map((line) => {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
fenced = !fenced;
|
||||
return line;
|
||||
}
|
||||
if (fenced) return line;
|
||||
return line.replace(
|
||||
/(?<!!)(\[[^\]]*\]\()([^)\s]*#[^)\s]+)(\))/g,
|
||||
(match, open, target, close) => {
|
||||
const file = linkedDocsFile(target, currentFile, lang);
|
||||
const entry = file && anchors.get(file);
|
||||
if (!entry) return match;
|
||||
const hashIndex = target.indexOf('#');
|
||||
const hash = decodeURIComponent(target.slice(hashIndex + 1));
|
||||
if (entry.ids.has(hash) || !entry.map.has(hash)) return match;
|
||||
return `${open}${target.slice(0, hashIndex + 1)}${entry.map.get(hash)}${close}`;
|
||||
}
|
||||
);
|
||||
})
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
export function brokenAnchors(markdown, currentFile, lang, anchors) {
|
||||
const broken = [];
|
||||
let fenced = false;
|
||||
for (const line of markdown.split('\n')) {
|
||||
if (/^\s*(```|~~~)/.test(line)) {
|
||||
fenced = !fenced;
|
||||
continue;
|
||||
}
|
||||
if (fenced) continue;
|
||||
for (const match of line.matchAll(
|
||||
/(?<!!)\[[^\]]*\]\(([^)\s]*#[^)\s]+)\)/g
|
||||
)) {
|
||||
const target = match[1];
|
||||
const file = linkedDocsFile(target, currentFile, lang);
|
||||
const entry = file && anchors.get(file);
|
||||
if (!entry) continue;
|
||||
const hash = decodeURIComponent(target.slice(target.indexOf('#') + 1));
|
||||
if (!entry.ids.has(hash)) broken.push(target);
|
||||
}
|
||||
}
|
||||
return broken;
|
||||
}
|
||||
|
||||
export function summarize(markdown) {
|
||||
let body = markdown;
|
||||
const frontmatterKeys = [];
|
||||
if (markdown.startsWith('---')) {
|
||||
const end = markdown.indexOf('\n---', 3);
|
||||
if (end > 0) {
|
||||
const frontmatter = markdown.slice(3, end);
|
||||
for (const line of frontmatter.split('\n')) {
|
||||
const key = line.match(/^([A-Za-z_][\w-]*):/);
|
||||
if (key) frontmatterKeys.push(key[1]);
|
||||
}
|
||||
body = markdown.slice(end + 4);
|
||||
}
|
||||
}
|
||||
const headings = [];
|
||||
const fences = [];
|
||||
const links = [];
|
||||
const containers = [];
|
||||
const tables = [];
|
||||
let fenced = null;
|
||||
let current = null;
|
||||
let tableRows = 0;
|
||||
for (const line of body.split('\n')) {
|
||||
const fence = line.match(/^\s*(```|~~~)/);
|
||||
if (fence) {
|
||||
if (fenced === null) {
|
||||
fenced = { marker: fence[1], lines: [line.trim()] };
|
||||
} else {
|
||||
fenced.lines.push(line.trim());
|
||||
fences.push(fenced.lines.join('\n'));
|
||||
fenced = null;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (fenced) {
|
||||
fenced.lines.push(line);
|
||||
continue;
|
||||
}
|
||||
const heading = line.match(/^(#{1,6})\s/);
|
||||
if (heading) headings.push(heading[1].length);
|
||||
const container = line.match(/^:::\s*([a-z-]+)?/);
|
||||
if (container) containers.push(container[1] || 'end');
|
||||
if (/^\s*\|/.test(line)) {
|
||||
tableRows += 1;
|
||||
current = true;
|
||||
} else if (current) {
|
||||
tables.push(tableRows);
|
||||
tableRows = 0;
|
||||
current = null;
|
||||
}
|
||||
for (const match of line.matchAll(/!?\[[^\]]*\]\(([^)\s]+)[^)]*\)/g)) {
|
||||
links.push(match[1]);
|
||||
}
|
||||
}
|
||||
if (current) tables.push(tableRows);
|
||||
return {
|
||||
frontmatterKeys,
|
||||
headings,
|
||||
fences,
|
||||
links: links.sort(),
|
||||
containers,
|
||||
tables,
|
||||
};
|
||||
}
|
||||
|
||||
function sameJson(a, b) {
|
||||
return JSON.stringify(a) === JSON.stringify(b);
|
||||
}
|
||||
|
||||
export function compareStructure(source, translated, lang) {
|
||||
const problems = [];
|
||||
const a = summarize(source);
|
||||
const b = summarize(translated);
|
||||
if (!sameJson(a.frontmatterKeys, b.frontmatterKeys)) {
|
||||
problems.push('frontmatter keys differ');
|
||||
}
|
||||
if (!sameJson(a.headings, b.headings)) problems.push('heading levels differ');
|
||||
if (!sameJson(a.fences, b.fences)) problems.push('code blocks differ');
|
||||
if (!sameJson(a.containers, b.containers)) problems.push('containers differ');
|
||||
if (!sameJson(a.tables, b.tables)) problems.push('table sizes differ');
|
||||
const withoutAnchor = (link) => link.replace(/#.*$/, '');
|
||||
const localizedLinks = (links) =>
|
||||
links
|
||||
.map((link) =>
|
||||
link.startsWith(`/${lang}/`) ? link.slice(lang.length + 1) : link
|
||||
)
|
||||
.map(withoutAnchor)
|
||||
.sort();
|
||||
if (!sameJson(a.links.map(withoutAnchor).sort(), localizedLinks(b.links))) {
|
||||
problems.push('link targets differ');
|
||||
}
|
||||
if (
|
||||
/—/.test(
|
||||
translated
|
||||
.replace(/^(```|~~~)[\s\S]*?^\1/gm, '')
|
||||
.replace(/`[^`\n]*`/g, '')
|
||||
)
|
||||
) {
|
||||
problems.push('contains an em dash');
|
||||
}
|
||||
if (translated.trim() === source.trim())
|
||||
problems.push('identical to English');
|
||||
const prose = proseOf(translated);
|
||||
const proseWords = wordCount(prose);
|
||||
const sourceWords = wordCount(proseOf(source));
|
||||
if (sourceWords > 40) {
|
||||
const range = SCRIPT_RANGES[lang];
|
||||
if (range && !range.test(prose)) problems.push('not in the target script');
|
||||
const tokens = prose
|
||||
.toLowerCase()
|
||||
.split(/[^\p{L}]+/u)
|
||||
.filter(Boolean);
|
||||
if (!range && tokens.length >= 40) {
|
||||
const hits = tokens.filter((token) => ENGLISH_MARKERS.has(token)).length;
|
||||
if (hits / tokens.length >= 0.08) problems.push('looks like English');
|
||||
}
|
||||
const unspaced = ['ja', 'zh', 'zh-TW'].includes(lang);
|
||||
const ratio = unspaced
|
||||
? prose.replace(/\s+/g, '').length /
|
||||
proseOf(source).replace(/\s+/g, '').length
|
||||
: proseWords / sourceWords;
|
||||
if (unspaced ? ratio < 0.15 || ratio > 1.5 : ratio < 0.4 || ratio > 3)
|
||||
problems.push(`length ratio ${ratio.toFixed(2)}`);
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
function groupsCommand(outFile, wordsPerGroup) {
|
||||
const files = listSourceFiles();
|
||||
const groups = [];
|
||||
let current = [];
|
||||
let words = 0;
|
||||
for (const file of files) {
|
||||
const count = wordCount(
|
||||
fs.readFileSync(path.join(DOCS_DIR, file), 'utf-8')
|
||||
);
|
||||
if (current.length > 0 && words + count > wordsPerGroup) {
|
||||
groups.push(current);
|
||||
current = [];
|
||||
words = 0;
|
||||
}
|
||||
current.push(file);
|
||||
words += count;
|
||||
}
|
||||
if (current.length > 0) groups.push(current);
|
||||
fs.mkdirSync(path.dirname(outFile), { recursive: true });
|
||||
fs.writeFileSync(outFile, JSON.stringify(groups, null, 2) + '\n');
|
||||
console.log(`${files.length} files in ${groups.length} groups`);
|
||||
}
|
||||
|
||||
const ENGLISH_ONLY_DOCS = new Set(['licensing.md']);
|
||||
|
||||
function checkCommand(lang, only) {
|
||||
const files = only.length > 0 ? only : listSourceFiles();
|
||||
const results = [];
|
||||
let missing = 0;
|
||||
const anchors = anchorsFor(translatedDocs(lang));
|
||||
for (const file of files) {
|
||||
const translatedPath = path.join(DOCS_DIR, lang, file);
|
||||
if (!fs.existsSync(translatedPath)) {
|
||||
missing += 1;
|
||||
results.push({ file, problems: ['missing'] });
|
||||
continue;
|
||||
}
|
||||
if (ENGLISH_ONLY_DOCS.has(file)) continue;
|
||||
const translated = fs.readFileSync(translatedPath, 'utf-8');
|
||||
const problems = compareStructure(
|
||||
fs.readFileSync(path.join(DOCS_DIR, file), 'utf-8'),
|
||||
translated,
|
||||
lang
|
||||
);
|
||||
const broken = brokenAnchors(translated, file, lang, anchors);
|
||||
if (broken.length > 0) {
|
||||
problems.push(`broken anchor links: ${broken.join(', ')}`);
|
||||
}
|
||||
if (/^#{1,6} .*\{#\\/m.test(translated)) {
|
||||
problems.push('escaped heading anchor');
|
||||
}
|
||||
if (problems.length > 0) results.push({ file, problems });
|
||||
}
|
||||
const bad = results.filter((entry) => !entry.problems.includes('missing'));
|
||||
console.log(
|
||||
`${lang}: ${files.length - missing}/${files.length} files present, ${bad.length} with problems`
|
||||
);
|
||||
if (results.length > 0)
|
||||
console.log(JSON.stringify(results.slice(0, 40), null, 2));
|
||||
process.exitCode = bad.length > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
function translatedDocs(lang) {
|
||||
const docs = new Map();
|
||||
for (const file of listSourceFiles()) {
|
||||
const target = path.join(DOCS_DIR, lang, file);
|
||||
if (!fs.existsSync(target)) continue;
|
||||
docs.set(file, {
|
||||
target,
|
||||
source: fs.readFileSync(path.join(DOCS_DIR, file), 'utf-8'),
|
||||
translated: fs.readFileSync(target, 'utf-8'),
|
||||
});
|
||||
}
|
||||
return docs;
|
||||
}
|
||||
|
||||
function anchorsFor(docs) {
|
||||
const anchors = new Map();
|
||||
for (const [file, doc] of docs) {
|
||||
anchors.set(file, anchorMap(doc.source, doc.translated));
|
||||
}
|
||||
return anchors;
|
||||
}
|
||||
|
||||
function localizeCommand(lang) {
|
||||
let changed = 0;
|
||||
for (const file of ENGLISH_ONLY_DOCS) {
|
||||
const target = path.join(DOCS_DIR, lang, file);
|
||||
fs.mkdirSync(path.dirname(target), { recursive: true });
|
||||
fs.copyFileSync(path.join(DOCS_DIR, file), target);
|
||||
}
|
||||
const docs = translatedDocs(lang);
|
||||
const originals = new Map();
|
||||
for (const [file, doc] of docs) {
|
||||
originals.set(file, doc.translated);
|
||||
doc.translated = quoteFrontmatter(
|
||||
removeGeneratedAnchors(doc.source, localizeLinks(doc.translated, lang))
|
||||
);
|
||||
}
|
||||
const anchors = anchorsFor(docs);
|
||||
for (const [file, doc] of docs) {
|
||||
const after = translateAnchors(doc.translated, file, lang, anchors);
|
||||
if (after !== originals.get(file)) {
|
||||
fs.writeFileSync(doc.target, after);
|
||||
changed += 1;
|
||||
}
|
||||
}
|
||||
console.log(`${lang}: localized links and anchors in ${changed} files`);
|
||||
}
|
||||
|
||||
function labelsCheckCommand(lang) {
|
||||
const sourceFile = path.join(DOCS_DIR, '.vitepress/i18n/source.json');
|
||||
const source = JSON.parse(fs.readFileSync(sourceFile, 'utf-8'));
|
||||
const targetFile = path.join(DOCS_DIR, `.vitepress/i18n/${lang}.json`);
|
||||
if (!fs.existsSync(targetFile)) {
|
||||
console.log(`${lang}: labels file is missing`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const labels = JSON.parse(fs.readFileSync(targetFile, 'utf-8'));
|
||||
const problems = [];
|
||||
for (const key of source) {
|
||||
const value = labels[key];
|
||||
if (typeof value !== 'string' || value.trim() === '') {
|
||||
problems.push({ key, problem: 'missing or empty' });
|
||||
} else if (/\u2014/.test(value)) {
|
||||
problems.push({ key, problem: 'contains an em dash' });
|
||||
}
|
||||
}
|
||||
for (const key of Object.keys(labels)) {
|
||||
if (!source.includes(key)) problems.push({ key, problem: 'unknown key' });
|
||||
}
|
||||
console.log(
|
||||
`${lang}: ${source.length - problems.length}/${source.length} labels ok`
|
||||
);
|
||||
if (problems.length > 0)
|
||||
console.log(JSON.stringify(problems.slice(0, 30), null, 2));
|
||||
process.exitCode = problems.length > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
function statusCommand() {
|
||||
const files = listSourceFiles();
|
||||
for (const lang of languages()) {
|
||||
const present = files.filter((file) =>
|
||||
fs.existsSync(path.join(DOCS_DIR, lang, file))
|
||||
).length;
|
||||
console.log(`${lang.padEnd(6)} ${present}/${files.length}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const [command, ...args] = process.argv.slice(2);
|
||||
if (command === 'groups') groupsCommand(args[0], Number(args[1] || 6000));
|
||||
else if (command === 'check') checkCommand(args[0], args.slice(1));
|
||||
else if (command === 'localize') localizeCommand(args[0]);
|
||||
else if (command === 'labels') labelsCheckCommand(args[0]);
|
||||
else if (command === 'status') statusCommand();
|
||||
else {
|
||||
console.error(
|
||||
'Usage: docs-translations.mjs groups <outFile> [words] | check <lang> [files...] | localize <lang> | labels <lang> | status'
|
||||
);
|
||||
process.exitCode = 2;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { ENGLISH_ONLY_PAGES, processPage } from './content-segments.mjs';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const outIndex = args.indexOf('--out');
|
||||
const outFile = path.resolve(
|
||||
outIndex >= 0
|
||||
? args.splice(outIndex, 2)[1]
|
||||
: path.join(__dirname, '../translations/source.json')
|
||||
);
|
||||
const distDirs = (
|
||||
args.length > 0 ? args : [path.join(__dirname, '../dist')]
|
||||
).map((dir) => path.resolve(dir));
|
||||
|
||||
const existing = fs.existsSync(outFile)
|
||||
? JSON.parse(fs.readFileSync(outFile, 'utf-8'))
|
||||
: {};
|
||||
|
||||
const segments = new Map();
|
||||
const pageIds = new Map();
|
||||
let pageCount = 0;
|
||||
|
||||
for (const distDir of distDirs) {
|
||||
const pages = fs
|
||||
.readdirSync(distDir)
|
||||
.filter((file) => file.endsWith('.html'))
|
||||
.sort();
|
||||
pageCount += pages.length;
|
||||
for (const file of pages) {
|
||||
const pageName = file.replace(/\.html$/, '');
|
||||
const dom = new JSDOM(fs.readFileSync(path.join(distDir, file), 'utf-8'));
|
||||
processPage(dom.window.document, pageName, {
|
||||
record(id, text, tag) {
|
||||
const entry = segments.get(id) || { text, tags: new Set(), pages: [] };
|
||||
entry.tags.add(tag);
|
||||
if (!entry.pages.includes(pageName)) entry.pages.push(pageName);
|
||||
segments.set(id, entry);
|
||||
if (!ENGLISH_ONLY_PAGES.has(pageName)) {
|
||||
if (!pageIds.has(pageName)) pageIds.set(pageName, new Set());
|
||||
pageIds.get(pageName).add(id);
|
||||
}
|
||||
},
|
||||
});
|
||||
dom.window.close();
|
||||
}
|
||||
}
|
||||
|
||||
const result = {};
|
||||
for (const id of [...segments.keys()].sort()) {
|
||||
const entry = segments.get(id);
|
||||
result[id] = {
|
||||
text: entry.text,
|
||||
tags: [...entry.tags].sort(),
|
||||
pages: entry.pages.slice(0, 3),
|
||||
pageCount: entry.pages.length,
|
||||
};
|
||||
}
|
||||
|
||||
fs.mkdirSync(path.dirname(outFile), { recursive: true });
|
||||
fs.writeFileSync(outFile, JSON.stringify(result, null, 2) + '\n');
|
||||
|
||||
const pageIndex = {};
|
||||
for (const page of [...pageIds.keys()].sort()) {
|
||||
pageIndex[page] = [...pageIds.get(page)].sort();
|
||||
}
|
||||
fs.writeFileSync(
|
||||
path.join(path.dirname(outFile), 'pages.json'),
|
||||
JSON.stringify(pageIndex) + '\n'
|
||||
);
|
||||
|
||||
const added = Object.keys(result).filter((id) => !existing[id]).length;
|
||||
const removed = Object.keys(existing).filter((id) => !result[id]).length;
|
||||
const words = Object.values(result).reduce(
|
||||
(sum, entry) => sum + entry.text.split(/\s+/).length,
|
||||
0
|
||||
);
|
||||
console.log(
|
||||
`Extracted ${Object.keys(result).length} unique segments (${words} words) from ${pageCount} pages. Added ${added}, removed ${removed}.`
|
||||
);
|
||||
@@ -0,0 +1,28 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { nav, sidebar } from '../docs/.vitepress/navigation.mjs';
|
||||
import {
|
||||
DESCRIPTION,
|
||||
FOOTER_MESSAGE,
|
||||
THEME_LABELS,
|
||||
} from '../docs/.vitepress/i18n/theme-labels.mjs';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const OUT_FILE = path.resolve(__dirname, '../docs/.vitepress/i18n/source.json');
|
||||
|
||||
const labels = new Set([DESCRIPTION, FOOTER_MESSAGE, ...THEME_LABELS]);
|
||||
|
||||
function collect(items) {
|
||||
for (const item of items) {
|
||||
if (item.text) labels.add(item.text);
|
||||
if (item.items) collect(item.items);
|
||||
}
|
||||
}
|
||||
|
||||
collect(nav);
|
||||
Object.values(sidebar).forEach(collect);
|
||||
|
||||
const sorted = [...labels].sort((a, b) => a.localeCompare(b));
|
||||
fs.writeFileSync(OUT_FILE, JSON.stringify(sorted, null, 2) + '\n');
|
||||
console.log(`Wrote ${sorted.length} docs labels to ${OUT_FILE}`);
|
||||
@@ -0,0 +1,142 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import ts from 'typescript';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '../src/js');
|
||||
const OUT_FILE = path.resolve(__dirname, '../translations/ui-source.json');
|
||||
const SKIP_DIRS = new Set(['editcore', 'types', 'i18n', 'config']);
|
||||
const HELPER_ARGUMENTS = {
|
||||
showAlert: 2,
|
||||
showLoader: 1,
|
||||
showAlertModal: 2,
|
||||
showModal: 2,
|
||||
};
|
||||
const LETTERS = /\p{L}{2}/u;
|
||||
|
||||
export function extractUiStrings() {
|
||||
const files = [];
|
||||
(function walk(dir) {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (!SKIP_DIRS.has(entry.name)) walk(full);
|
||||
} else if (/\.ts$/.test(entry.name)) {
|
||||
files.push(full);
|
||||
}
|
||||
}
|
||||
})(ROOT);
|
||||
|
||||
const strings = new Map();
|
||||
|
||||
const formatControlLabel = (key) =>
|
||||
key
|
||||
.replace(/([A-Z])/g, ' $1')
|
||||
.replace(/^./, (c) => c.toUpperCase())
|
||||
.trim();
|
||||
|
||||
function add(text, file) {
|
||||
if (!LETTERS.test(text)) return;
|
||||
const entry = strings.get(text) || { files: [] };
|
||||
const relative = path.relative(ROOT, file);
|
||||
if (!entry.files.includes(relative)) entry.files.push(relative);
|
||||
strings.set(text, entry);
|
||||
}
|
||||
|
||||
function isStringLike(node) {
|
||||
return ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node);
|
||||
}
|
||||
|
||||
function collectLiterals(node, file) {
|
||||
if (isStringLike(node)) {
|
||||
add(node.text, file);
|
||||
} else if (ts.isParenthesizedExpression(node)) {
|
||||
collectLiterals(node.expression, file);
|
||||
} else if (ts.isConditionalExpression(node)) {
|
||||
collectLiterals(node.whenTrue, file);
|
||||
collectLiterals(node.whenFalse, file);
|
||||
} else if (
|
||||
ts.isBinaryExpression(node) &&
|
||||
(node.operatorToken.kind === ts.SyntaxKind.BarBarToken ||
|
||||
node.operatorToken.kind === ts.SyntaxKind.QuestionQuestionToken)
|
||||
) {
|
||||
collectLiterals(node.left, file);
|
||||
collectLiterals(node.right, file);
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of files) {
|
||||
const source = fs.readFileSync(file, 'utf8');
|
||||
const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
||||
const visit = (node) => {
|
||||
if (ts.isCallExpression(node)) {
|
||||
const callee = ts.isIdentifier(node.expression)
|
||||
? node.expression.text
|
||||
: ts.isPropertyAccessExpression(node.expression)
|
||||
? node.expression.name.text
|
||||
: '';
|
||||
if (
|
||||
(callee === 'trText' || callee === 'trHtml') &&
|
||||
node.arguments.length > 0
|
||||
) {
|
||||
collectLiterals(node.arguments[0], file);
|
||||
} else if (callee in HELPER_ARGUMENTS) {
|
||||
node.arguments
|
||||
.slice(0, HELPER_ARGUMENTS[callee])
|
||||
.forEach((argument) => collectLiterals(argument, file));
|
||||
} else if (
|
||||
callee === 'addControl' &&
|
||||
node.arguments.length > 0 &&
|
||||
isStringLike(node.arguments[0])
|
||||
) {
|
||||
add(formatControlLabel(node.arguments[0].text), file);
|
||||
}
|
||||
}
|
||||
const relative = path.relative(ROOT, file);
|
||||
if (
|
||||
relative === path.join('workflow', 'nodes', 'registry.ts') &&
|
||||
ts.isPropertyAssignment(node) &&
|
||||
ts.isIdentifier(node.name) &&
|
||||
['label', 'description'].includes(node.name.text) &&
|
||||
isStringLike(node.initializer)
|
||||
) {
|
||||
add(node.initializer.text, file);
|
||||
}
|
||||
if (
|
||||
relative === path.join('workflow', 'nodes', 'control-specs.ts') &&
|
||||
ts.isArrayLiteralExpression(node) &&
|
||||
node.elements.length === 2 &&
|
||||
node.elements.every(isStringLike)
|
||||
) {
|
||||
add(node.elements[1].text, file);
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
visit(sf);
|
||||
}
|
||||
|
||||
const result = {};
|
||||
for (const text of [...strings.keys()].sort()) {
|
||||
const entry = strings.get(text);
|
||||
result[text] = {
|
||||
files: entry.files.slice(0, 3),
|
||||
fileCount: entry.files.length,
|
||||
};
|
||||
}
|
||||
return { result, fileCount: files.length };
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const { result, fileCount } = extractUiStrings();
|
||||
fs.mkdirSync(path.dirname(OUT_FILE), { recursive: true });
|
||||
fs.writeFileSync(OUT_FILE, JSON.stringify(result, null, 2) + '\n');
|
||||
|
||||
const words = Object.keys(result).reduce(
|
||||
(sum, text) => sum + text.split(/\s+/).length,
|
||||
0
|
||||
);
|
||||
console.log(
|
||||
`Extracted ${Object.keys(result).length} UI strings (${words} words) from ${fileCount} files.`
|
||||
);
|
||||
}
|
||||
+113
-53
@@ -3,12 +3,18 @@ import path from 'path';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { serializeJsonLd } from './json-ld.mjs';
|
||||
import { ENGLISH_ONLY_PAGES, processPage } from './content-segments.mjs';
|
||||
import { completeDocsLanguages } from './docs-languages.mjs';
|
||||
import { createTranslationChecker } from './page-translation.mjs';
|
||||
import { canonicalBasePath } from './indexing-policy.mjs';
|
||||
import { applyServerTranslations } from './server-translations.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
const DIST_DIR = path.resolve(__dirname, '../dist');
|
||||
const LOCALES_DIR = path.resolve(__dirname, '../public/locales');
|
||||
const CONTENT_DIR = path.resolve(__dirname, '../translations');
|
||||
const SITE_URL = (process.env.SITE_URL || 'https://www.bentopdf.com').replace(
|
||||
/\/+$/,
|
||||
''
|
||||
@@ -19,6 +25,14 @@ const languages = fs.readdirSync(LOCALES_DIR).filter((file) => {
|
||||
return fs.statSync(path.join(LOCALES_DIR, file)).isDirectory();
|
||||
});
|
||||
|
||||
const OFFICIAL_DOCS_URL = 'https://www.bentopdf.com/docs';
|
||||
|
||||
const docsLanguages = completeDocsLanguages(
|
||||
path.resolve(__dirname, '../docs'),
|
||||
languages.filter((lang) => lang !== 'en'),
|
||||
process.env.DOCS_LANGUAGES
|
||||
).enabled;
|
||||
|
||||
const toCamelCase = (str) => {
|
||||
return str.replace(/-([a-z])/g, (g) => g[1].toUpperCase());
|
||||
};
|
||||
@@ -34,7 +48,11 @@ function loadAllTranslations() {
|
||||
if (lang === 'en') continue;
|
||||
const commonPath = path.join(LOCALES_DIR, `${lang}/common.json`);
|
||||
const toolsPath = path.join(LOCALES_DIR, `${lang}/tools.json`);
|
||||
const contentPath = path.join(CONTENT_DIR, lang, 'content.json');
|
||||
translations[lang] = {
|
||||
content: fs.existsSync(contentPath)
|
||||
? JSON.parse(fs.readFileSync(contentPath, 'utf-8'))
|
||||
: {},
|
||||
common: fs.existsSync(commonPath)
|
||||
? JSON.parse(fs.readFileSync(commonPath, 'utf-8'))
|
||||
: {},
|
||||
@@ -54,54 +72,18 @@ function loadEnglishTools() {
|
||||
|
||||
const ENGLISH_TOOLS = loadEnglishTools();
|
||||
|
||||
const contentTotals = {};
|
||||
|
||||
// TODO@ALAM: Let users build only a single language
|
||||
function buildUrl(langPrefix, pagePath) {
|
||||
const parts = [SITE_URL];
|
||||
if (BASE_PATH && BASE_PATH !== '') parts.push(BASE_PATH.replace(/^\//, ''));
|
||||
const canonicalBase = canonicalBasePath();
|
||||
if (canonicalBase) parts.push(canonicalBase.replace(/^\//, ''));
|
||||
if (langPrefix) parts.push(langPrefix);
|
||||
if (pagePath) parts.push(pagePath.replace(/^\//, ''));
|
||||
return parts.filter(Boolean).join('/').replace(/\/+$/, '') || SITE_URL;
|
||||
}
|
||||
|
||||
function resolveTranslation(langResources, key) {
|
||||
let namespace = 'common';
|
||||
let keyPath = key;
|
||||
const colonIndex = key.indexOf(':');
|
||||
if (colonIndex !== -1) {
|
||||
namespace = key.slice(0, colonIndex);
|
||||
keyPath = key.slice(colonIndex + 1);
|
||||
}
|
||||
let node = langResources[namespace];
|
||||
for (const segment of keyPath.split('.')) {
|
||||
if (node == null || typeof node !== 'object') return null;
|
||||
node = node[segment];
|
||||
}
|
||||
return typeof node === 'string' && node ? node : null;
|
||||
}
|
||||
|
||||
function applyServerTranslations(document, langResources) {
|
||||
document.querySelectorAll('[data-i18n]').forEach((element) => {
|
||||
const key = element.getAttribute('data-i18n');
|
||||
if (!key) return;
|
||||
const translation = resolveTranslation(langResources, key);
|
||||
if (translation) element.textContent = translation;
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-i18n-placeholder]').forEach((element) => {
|
||||
const key = element.getAttribute('data-i18n-placeholder');
|
||||
if (!key) return;
|
||||
const translation = resolveTranslation(langResources, key);
|
||||
if (translation) element.setAttribute('placeholder', translation);
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-i18n-title]').forEach((element) => {
|
||||
const key = element.getAttribute('data-i18n-title');
|
||||
if (!key) return;
|
||||
const translation = resolveTranslation(langResources, key);
|
||||
if (translation) element.setAttribute('title', translation);
|
||||
});
|
||||
}
|
||||
|
||||
const ORGANIZATION_LD_MARKER = 'data-bentopdf-organization';
|
||||
|
||||
function injectOrganizationLd(document) {
|
||||
@@ -158,7 +140,13 @@ function removeStaticBreadcrumbLd(document) {
|
||||
}
|
||||
}
|
||||
|
||||
function injectToolBreadcrumb(document, lang, toolName, toolUrl) {
|
||||
function injectToolBreadcrumb(
|
||||
document,
|
||||
lang,
|
||||
toolName,
|
||||
toolUrl,
|
||||
label = 'Breadcrumb'
|
||||
) {
|
||||
const h1 = document.querySelector('h1[data-i18n^="tools:"]');
|
||||
if (!h1) return;
|
||||
if (document.querySelector(`[${BREADCRUMB_MARKER}]`)) return;
|
||||
@@ -167,8 +155,9 @@ function injectToolBreadcrumb(document, lang, toolName, toolUrl) {
|
||||
const homeUrl = buildUrl(lang === 'en' ? '' : lang, '');
|
||||
|
||||
const nav = document.createElement('nav');
|
||||
nav.setAttribute('aria-label', 'Breadcrumb');
|
||||
nav.setAttribute('aria-label', label);
|
||||
nav.setAttribute(BREADCRUMB_MARKER, '');
|
||||
nav.setAttribute('data-no-i18n', '');
|
||||
nav.className = 'text-sm text-gray-400 mb-4';
|
||||
|
||||
const homeLink = document.createElement('a');
|
||||
@@ -229,14 +218,20 @@ function translationKeyForFile(filenameNoExt) {
|
||||
return KEY_MAPPING[filenameNoExt] || toCamelCase(filenameNoExt);
|
||||
}
|
||||
|
||||
let isPageTranslated = null;
|
||||
|
||||
function translatedLangsForFile(filenameNoExt, translations) {
|
||||
const translationKey = translationKeyForFile(filenameNoExt);
|
||||
return languages.filter(
|
||||
(l) =>
|
||||
l === 'en' ||
|
||||
filenameNoExt === 'index' ||
|
||||
Boolean(translations[l] && translations[l].tools[translationKey])
|
||||
);
|
||||
if (!isPageTranslated) {
|
||||
const toolsByLang = {};
|
||||
for (const lang of languages) {
|
||||
toolsByLang[lang] = translations[lang] ? translations[lang].tools : {};
|
||||
}
|
||||
isPageTranslated = createTranslationChecker(
|
||||
toolsByLang,
|
||||
translationKeyForFile
|
||||
);
|
||||
}
|
||||
return languages.filter((l) => isPageTranslated(filenameNoExt, l));
|
||||
}
|
||||
|
||||
function processFileForLanguage(
|
||||
@@ -257,8 +252,28 @@ function processFileForLanguage(
|
||||
|
||||
document.documentElement.lang = lang;
|
||||
document.documentElement.dir = ['ar', 'fa'].includes(lang) ? 'rtl' : 'ltr';
|
||||
document.querySelector('script[data-language-redirect]')?.remove();
|
||||
if (ENGLISH_ONLY_PAGES.has(filenameNoExt)) {
|
||||
const englishContent = document.getElementById('app');
|
||||
englishContent?.setAttribute('lang', 'en');
|
||||
englishContent?.setAttribute('dir', 'ltr');
|
||||
}
|
||||
|
||||
const metaContent = (selector) =>
|
||||
document.querySelector(selector)?.getAttribute('content') ?? null;
|
||||
const englishMeta = {
|
||||
title: document.title,
|
||||
description: metaContent('meta[name="description"]'),
|
||||
};
|
||||
|
||||
applyServerTranslations(document, translations[lang]);
|
||||
const contentStats = processPage(document, filenameNoExt, {
|
||||
translate: (id) => translations[lang].content[id] || null,
|
||||
});
|
||||
contentTotals[lang] ??= { found: 0, translated: 0, invalid: 0 };
|
||||
contentTotals[lang].found += contentStats.found;
|
||||
contentTotals[lang].translated += contentStats.translated;
|
||||
contentTotals[lang].invalid += contentStats.invalid.length;
|
||||
|
||||
let title = null;
|
||||
let description = null;
|
||||
@@ -272,7 +287,11 @@ function processFileForLanguage(
|
||||
description = tools[translationKey].subtitle;
|
||||
}
|
||||
|
||||
if (title) {
|
||||
const titleTranslated = document.title !== englishMeta.title;
|
||||
const descriptionTranslated =
|
||||
metaContent('meta[name="description"]') !== englishMeta.description;
|
||||
|
||||
if (title && !titleTranslated) {
|
||||
document.title = title;
|
||||
const metaTitle = document.querySelector('meta[property="og:title"]');
|
||||
if (metaTitle) metaTitle.content = title;
|
||||
@@ -282,7 +301,7 @@ function processFileForLanguage(
|
||||
if (metaTwitterTitle) metaTwitterTitle.content = title;
|
||||
}
|
||||
|
||||
if (description) {
|
||||
if (description && !descriptionTranslated) {
|
||||
const metaDesc = document.querySelector('meta[name="description"]');
|
||||
if (metaDesc) metaDesc.content = description;
|
||||
const metaOgDesc = document.querySelector(
|
||||
@@ -337,7 +356,28 @@ function processFileForLanguage(
|
||||
|
||||
const localizedToolName = resolveToolName(translationKey, tools);
|
||||
if (localizedToolName) {
|
||||
injectToolBreadcrumb(document, lang, localizedToolName, localizedUrl);
|
||||
injectToolBreadcrumb(
|
||||
document,
|
||||
lang,
|
||||
localizedToolName,
|
||||
localizedUrl,
|
||||
translations[lang].common?.nav?.breadcrumb || 'Breadcrumb'
|
||||
);
|
||||
}
|
||||
|
||||
if (docsLanguages.includes(lang)) {
|
||||
document.querySelectorAll('a[href]').forEach((link) => {
|
||||
const href = link.getAttribute('href');
|
||||
if (
|
||||
href === OFFICIAL_DOCS_URL ||
|
||||
href?.startsWith(`${OFFICIAL_DOCS_URL}/`)
|
||||
) {
|
||||
link.setAttribute(
|
||||
'href',
|
||||
href.replace(OFFICIAL_DOCS_URL, `${OFFICIAL_DOCS_URL}/${lang}`)
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const links = document.querySelectorAll('a[href]');
|
||||
@@ -360,7 +400,13 @@ function processFileForLanguage(
|
||||
|
||||
if (href.startsWith('/assets/') || href.includes('/assets/')) return;
|
||||
|
||||
if (href === '/blog' || href.startsWith('/blog/')) return;
|
||||
const hrefWithoutBase =
|
||||
BASE_PATH && href.startsWith(`${BASE_PATH}/`)
|
||||
? href.slice(BASE_PATH.length)
|
||||
: href;
|
||||
if (hrefWithoutBase === '/blog' || hrefWithoutBase.startsWith('/blog/')) {
|
||||
return;
|
||||
}
|
||||
|
||||
const langPrefixRegex = new RegExp(
|
||||
`^(${BASE_PATH})?/(${languages.join('|')})(/|$)`
|
||||
@@ -510,6 +556,20 @@ async function generateI18nPages() {
|
||||
updateEnglishFile(filePath, originalContent, translatedLangs);
|
||||
}
|
||||
|
||||
const covered = Object.entries(contentTotals).filter(
|
||||
([, totals]) => totals.translated > 0
|
||||
);
|
||||
if (covered.length > 0) {
|
||||
console.log(
|
||||
` Page content translated: ${covered
|
||||
.map(
|
||||
([lang, totals]) =>
|
||||
`${lang} ${Math.round((totals.translated / totals.found) * 100)}%${totals.invalid ? ` (${totals.invalid} rejected)` : ''}`
|
||||
)
|
||||
.join(', ')}`
|
||||
);
|
||||
}
|
||||
|
||||
console.log('✅ i18n pages generated successfully!');
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { OFFICIAL_SITE_URL, canonicalBasePath } from './indexing-policy.mjs';
|
||||
import { createTranslationChecker } from './page-translation.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
@@ -30,12 +32,10 @@ for (const lang of languages) {
|
||||
: {};
|
||||
}
|
||||
|
||||
const isPageTranslated = createTranslationChecker(toolsByLang, toCamelCase);
|
||||
|
||||
function alternateLangsFor(pageName) {
|
||||
if (pageName === 'index') return languages;
|
||||
const translationKey = toCamelCase(pageName);
|
||||
return languages.filter(
|
||||
(l) => l === 'en' || Boolean(toolsByLang[l][translationKey])
|
||||
);
|
||||
return languages.filter((l) => isPageTranslated(pageName, l));
|
||||
}
|
||||
|
||||
const PRIORITY_MAP = {
|
||||
@@ -78,10 +78,9 @@ function getPriority(pageName) {
|
||||
|
||||
function buildUrl(lang, pageName) {
|
||||
const pagePath = pageName === 'index' ? '' : pageName;
|
||||
if (lang === 'en') {
|
||||
return pagePath ? `${SITE_URL}/${pagePath}` : SITE_URL;
|
||||
}
|
||||
return pagePath ? `${SITE_URL}/${lang}/${pagePath}` : `${SITE_URL}/${lang}`;
|
||||
const root = `${SITE_URL}${canonicalBasePath()}`;
|
||||
const prefix = lang === 'en' ? root : `${root}/${lang}`;
|
||||
return pagePath ? `${prefix}/${pagePath}` : prefix;
|
||||
}
|
||||
|
||||
function generateSitemap() {
|
||||
@@ -170,7 +169,7 @@ function generateSitemap() {
|
||||
|
||||
const blogDir = path.join(DIST_DIR, 'blog');
|
||||
let blogCount = 0;
|
||||
if (fs.existsSync(blogDir)) {
|
||||
if (SITE_URL === OFFICIAL_SITE_URL && fs.existsSync(blogDir)) {
|
||||
const blogFiles = fs
|
||||
.readdirSync(blogDir)
|
||||
.filter((file) => file.endsWith('.html'))
|
||||
|
||||
@@ -111,7 +111,14 @@ function listDocsHtml(dir, prefix) {
|
||||
);
|
||||
} else if (entry.name.endsWith('.html') && entry.name !== '404.html') {
|
||||
const slug = entry.name.replace(/\.html$/, '');
|
||||
urls.push(slug === 'index' ? prefix : `${prefix}/${slug}`);
|
||||
const html = fs.readFileSync(path.join(dir, entry.name), 'utf-8');
|
||||
const canonical = html.match(
|
||||
/<link[^>]+rel="canonical"[^>]+href="([^"]+)"/
|
||||
);
|
||||
urls.push({
|
||||
url: slug === 'index' ? prefix : `${prefix}/${slug}`,
|
||||
canonical: canonical ? canonical[1] : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
@@ -132,7 +139,9 @@ function addDocsToSitemap() {
|
||||
let xml = fs.readFileSync(sitemapPath, 'utf-8');
|
||||
if (xml.includes(`<loc>${siteUrl}/docs</loc>`)) return;
|
||||
|
||||
const urls = listDocsHtml(targetDocsPath, '/docs');
|
||||
const urls = listDocsHtml(targetDocsPath, '/docs')
|
||||
.filter(({ url, canonical }) => canonical === `${siteUrl}${url}`)
|
||||
.map(({ url }) => url);
|
||||
const entries = urls
|
||||
.map(
|
||||
(url) =>
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
export const OFFICIAL_SITE_URL = 'https://www.bentopdf.com';
|
||||
|
||||
export function resolveSiteUrl() {
|
||||
return (process.env.SITE_URL || OFFICIAL_SITE_URL).replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
export function isOfficialSiteUrl(url = resolveSiteUrl()) {
|
||||
try {
|
||||
const { hostname } = new URL(url);
|
||||
return hostname === 'www.bentopdf.com' || hostname === 'bentopdf.com';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function canonicalBasePath() {
|
||||
if (isOfficialSiteUrl()) return '';
|
||||
return (process.env.BASE_URL || '/').replace(/\/$/, '');
|
||||
}
|
||||
|
||||
export function isIndexingBlocked() {
|
||||
return !isOfficialSiteUrl() && process.env.ALLOW_INDEXING !== 'true';
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const TRANSLATIONS_DIR = path.resolve(__dirname, '../translations');
|
||||
|
||||
export const COVERAGE_THRESHOLD = 0.8;
|
||||
|
||||
function readJson(file, fallback) {
|
||||
return fs.existsSync(file)
|
||||
? JSON.parse(fs.readFileSync(file, 'utf-8'))
|
||||
: fallback;
|
||||
}
|
||||
|
||||
export function createTranslationChecker(toolsByLang, keyFor) {
|
||||
const pageIndex = readJson(path.join(TRANSLATIONS_DIR, 'pages.json'), {});
|
||||
const idsByLang = new Map();
|
||||
const idsFor = (lang) => {
|
||||
if (!idsByLang.has(lang)) {
|
||||
const content = readJson(
|
||||
path.join(TRANSLATIONS_DIR, lang, 'content.json'),
|
||||
{}
|
||||
);
|
||||
idsByLang.set(
|
||||
lang,
|
||||
new Set(
|
||||
Object.keys(content).filter(
|
||||
(id) => typeof content[id] === 'string' && content[id].trim()
|
||||
)
|
||||
)
|
||||
);
|
||||
}
|
||||
return idsByLang.get(lang);
|
||||
};
|
||||
return (pageName, lang) => {
|
||||
if (lang === 'en' || pageName === 'index') return true;
|
||||
if (toolsByLang[lang] && toolsByLang[lang][keyFor(pageName)]) return true;
|
||||
const ids = pageIndex[pageName];
|
||||
if (!ids || ids.length === 0) return false;
|
||||
const translated = idsFor(lang);
|
||||
let covered = 0;
|
||||
for (const id of ids) if (translated.has(id)) covered += 1;
|
||||
return covered / ids.length >= COVERAGE_THRESHOLD;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,210 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import ts from 'typescript';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const SOURCE_ROOT = path.resolve(__dirname, '../src/js');
|
||||
const SKIPPED_DIRECTORIES = new Set(['editcore', 'types', 'i18n', 'config']);
|
||||
const EXEMPT_FILES = new Set(['logic/email-to-pdf.ts', 'utils/comic-info.ts']);
|
||||
const TRANSLATING_HELPERS = {
|
||||
showAlert: 2,
|
||||
showLoader: 1,
|
||||
showAlertModal: 2,
|
||||
showModal: 2,
|
||||
};
|
||||
const NATIVE_DIALOGS = { alert: 1, confirm: 1 };
|
||||
const HELPER_ARGUMENTS = { ...TRANSLATING_HELPERS, ...NATIVE_DIALOGS };
|
||||
const TEXT_PROPERTIES = new Set([
|
||||
'textContent',
|
||||
'title',
|
||||
'placeholder',
|
||||
'alt',
|
||||
'ariaLabel',
|
||||
]);
|
||||
const TEXT_ATTRIBUTES = new Set(['title', 'placeholder', 'aria-label', 'alt']);
|
||||
const LETTERS = /\p{L}{2}/u;
|
||||
const CSS_RULE = /[#.\w-]+\s*\{[^}]*:[^}]*\}/;
|
||||
const KEY_COMBINATION = /^(?:(?:Ctrl|Cmd|Alt|Shift|⌘|⌥|⇧)\s*\+\s*)+\S$/;
|
||||
|
||||
function sourceFiles(root) {
|
||||
const files = [];
|
||||
const walk = (dir) => {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
if (!SKIPPED_DIRECTORIES.has(entry.name)) walk(full);
|
||||
} else if (/\.ts$/.test(entry.name)) {
|
||||
files.push(full);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(root);
|
||||
return files;
|
||||
}
|
||||
|
||||
function isStringLike(node) {
|
||||
return ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node);
|
||||
}
|
||||
|
||||
function calleeName(call) {
|
||||
if (ts.isIdentifier(call.expression)) return call.expression.text;
|
||||
if (ts.isPropertyAccessExpression(call.expression)) {
|
||||
return call.expression.name.text;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
function rawLiterals(node, found = []) {
|
||||
if (isStringLike(node)) {
|
||||
if (LETTERS.test(node.text) && !KEY_COMBINATION.test(node.text)) {
|
||||
found.push(node);
|
||||
}
|
||||
} else if (ts.isTemplateExpression(node)) {
|
||||
const staticText = [
|
||||
node.head.text,
|
||||
...node.templateSpans.map((span) => span.literal.text),
|
||||
].join(' ');
|
||||
if (LETTERS.test(staticText)) found.push(node);
|
||||
} else if (ts.isParenthesizedExpression(node)) {
|
||||
rawLiterals(node.expression, found);
|
||||
} else if (ts.isConditionalExpression(node)) {
|
||||
rawLiterals(node.whenTrue, found);
|
||||
rawLiterals(node.whenFalse, found);
|
||||
} else if (ts.isBinaryExpression(node)) {
|
||||
const kind = node.operatorToken.kind;
|
||||
if (
|
||||
kind === ts.SyntaxKind.BarBarToken ||
|
||||
kind === ts.SyntaxKind.QuestionQuestionToken ||
|
||||
kind === ts.SyntaxKind.PlusToken
|
||||
) {
|
||||
rawLiterals(node.left, found);
|
||||
rawLiterals(node.right, found);
|
||||
}
|
||||
} else if (ts.isAsExpression(node) || ts.isNonNullExpression(node)) {
|
||||
rawLiterals(node.expression, found);
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
function isTranslatingCall(node) {
|
||||
const call = node.parent;
|
||||
return (
|
||||
call &&
|
||||
ts.isCallExpression(call) &&
|
||||
['trText', 'trHtml', 't', 'translate'].includes(calleeName(call))
|
||||
);
|
||||
}
|
||||
|
||||
function rawHtmlText(node) {
|
||||
let pieces;
|
||||
if (isStringLike(node)) {
|
||||
pieces = [node.text];
|
||||
} else if (ts.isTemplateExpression(node)) {
|
||||
pieces = [node.head.text, ...node.templateSpans.map((s) => s.literal.text)];
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
const html = pieces.join('\u0000');
|
||||
if (!/<[a-z][\w-]*[\s>/]/i.test(html)) return false;
|
||||
for (const match of html.matchAll(/>([^<>]+)(?=<)/g)) {
|
||||
const tagStart = html.lastIndexOf('<', match.index);
|
||||
const tag = html.slice(tagStart, match.index + 1);
|
||||
if (/data-i18n/.test(tag) || /^<\/?(script|style)/i.test(tag)) continue;
|
||||
const visible = match[1]
|
||||
.replace(/\u0000/g, ' ')
|
||||
.replace(/&\w+;|&#\d+;/g, ' ');
|
||||
if (LETTERS.test(visible)) return true;
|
||||
}
|
||||
for (const match of html.matchAll(
|
||||
/<[^<>]*?\s(?:title|placeholder|aria-label|alt)="([^"]*)"[^<>]*>/g
|
||||
)) {
|
||||
if (/data-i18n/.test(match[0])) continue;
|
||||
if (LETTERS.test(match[1].replace(/\u0000/g, ' '))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function scanRawUiText(root = SOURCE_ROOT) {
|
||||
const findings = [];
|
||||
for (const file of sourceFiles(root)) {
|
||||
if (EXEMPT_FILES.has(path.relative(root, file))) continue;
|
||||
const source = fs.readFileSync(file, 'utf8');
|
||||
const sf = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
||||
const report = (node, kind) => {
|
||||
const { line } = sf.getLineAndCharacterOfPosition(node.getStart(sf));
|
||||
findings.push({
|
||||
file: path.relative(root, file),
|
||||
line: line + 1,
|
||||
kind,
|
||||
text: node.getText(sf).replace(/\s+/g, ' ').slice(0, 100),
|
||||
});
|
||||
};
|
||||
const visit = (node) => {
|
||||
if (ts.isCallExpression(node)) {
|
||||
const name = calleeName(node);
|
||||
if (name in HELPER_ARGUMENTS) {
|
||||
node.arguments.slice(0, HELPER_ARGUMENTS[name]).forEach((argument) =>
|
||||
rawLiterals(argument)
|
||||
.filter(
|
||||
(raw) => !(name in TRANSLATING_HELPERS && isStringLike(raw))
|
||||
)
|
||||
.forEach((raw) => report(raw, name))
|
||||
);
|
||||
} else if (
|
||||
name === 'setAttribute' &&
|
||||
node.arguments.length === 2 &&
|
||||
isStringLike(node.arguments[0]) &&
|
||||
TEXT_ATTRIBUTES.has(node.arguments[0].text)
|
||||
) {
|
||||
rawLiterals(node.arguments[1]).forEach((raw) =>
|
||||
report(raw, 'setAttribute')
|
||||
);
|
||||
}
|
||||
}
|
||||
if (
|
||||
ts.isNewExpression(node) &&
|
||||
ts.isIdentifier(node.expression) &&
|
||||
node.expression.text === 'Option' &&
|
||||
node.arguments &&
|
||||
node.arguments.length > 0
|
||||
) {
|
||||
rawLiterals(node.arguments[0]).forEach((raw) => report(raw, 'Option'));
|
||||
}
|
||||
if (
|
||||
ts.isBinaryExpression(node) &&
|
||||
node.operatorToken.kind === ts.SyntaxKind.EqualsToken &&
|
||||
ts.isPropertyAccessExpression(node.left) &&
|
||||
TEXT_PROPERTIES.has(node.left.name.text)
|
||||
) {
|
||||
rawLiterals(node.right)
|
||||
.filter((raw) => !CSS_RULE.test(raw.getText()))
|
||||
.forEach((raw) => report(raw, node.left.name.text));
|
||||
}
|
||||
if (
|
||||
(isStringLike(node) || ts.isTemplateExpression(node)) &&
|
||||
!isTranslatingCall(node) &&
|
||||
rawHtmlText(node)
|
||||
) {
|
||||
report(node, 'html');
|
||||
}
|
||||
ts.forEachChild(node, visit);
|
||||
};
|
||||
visit(sf);
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const findings = scanRawUiText();
|
||||
const byKind = {};
|
||||
for (const finding of findings)
|
||||
byKind[finding.kind] = (byKind[finding.kind] || 0) + 1;
|
||||
console.log(`${findings.length} raw UI text sites`, JSON.stringify(byKind));
|
||||
for (const finding of findings.slice(0, Number(process.argv[2] || 25))) {
|
||||
console.log(
|
||||
`${finding.file}:${finding.line} [${finding.kind}] ${finding.text}`
|
||||
);
|
||||
}
|
||||
process.exitCode = findings.length > 0 ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import zlib from 'zlib';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const DIST_DIR = path.resolve(__dirname, '../dist');
|
||||
|
||||
function listHtml(dir) {
|
||||
return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) return listHtml(full);
|
||||
return entry.name.endsWith('.html') ? [full] : [];
|
||||
});
|
||||
}
|
||||
|
||||
let refreshed = 0;
|
||||
for (const file of listHtml(DIST_DIR)) {
|
||||
const brotli = `${file}.br`;
|
||||
const gzip = `${file}.gz`;
|
||||
if (!fs.existsSync(brotli) && !fs.existsSync(gzip)) continue;
|
||||
const html = fs.readFileSync(file);
|
||||
if (fs.existsSync(brotli)) {
|
||||
fs.writeFileSync(
|
||||
brotli,
|
||||
zlib.brotliCompressSync(html, {
|
||||
params: { [zlib.constants.BROTLI_PARAM_QUALITY]: 11 },
|
||||
})
|
||||
);
|
||||
}
|
||||
if (fs.existsSync(gzip)) {
|
||||
fs.writeFileSync(gzip, zlib.gzipSync(html, { level: 9 }));
|
||||
}
|
||||
refreshed++;
|
||||
}
|
||||
console.log(`Recompressed ${refreshed} precompressed HTML files.`);
|
||||
+11
-6
@@ -1,6 +1,8 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { canonicalBasePath, isIndexingBlocked } from './indexing-policy.mjs';
|
||||
import { createTranslationChecker } from './page-translation.mjs';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
@@ -90,6 +92,11 @@ for (const lang of LOCALE_DIRS) {
|
||||
: {};
|
||||
}
|
||||
|
||||
const isPageTranslated = createTranslationChecker(
|
||||
toolsByLang,
|
||||
(baseName) => KEY_MAPPING[baseName] || toCamelCase(baseName)
|
||||
);
|
||||
|
||||
function hasTrailingSlash(href) {
|
||||
const { pathname } = new URL(href);
|
||||
return pathname !== '/' && pathname.endsWith('/');
|
||||
@@ -107,15 +114,12 @@ function expectedCanonicalForFile(rel) {
|
||||
if (isBlogPage(rel)) {
|
||||
return `${OFFICIAL_SITE_URL}/blog${slug ? `/${slug}` : ''}`;
|
||||
}
|
||||
const translationKey = KEY_MAPPING[baseName] || toCamelCase(baseName);
|
||||
const segments = [SITE_URL];
|
||||
if (BASE_PATH) segments.push(BASE_PATH.replace(/^\//, ''));
|
||||
const canonicalBase = canonicalBasePath();
|
||||
if (canonicalBase) segments.push(canonicalBase.replace(/^\//, ''));
|
||||
for (const dir of parts) {
|
||||
if (LOCALE_DIRS.has(dir)) {
|
||||
const translated =
|
||||
baseName === 'index' ||
|
||||
Boolean(toolsByLang[dir] && toolsByLang[dir][translationKey]);
|
||||
if (translated) segments.push(dir);
|
||||
if (isPageTranslated(baseName, dir)) segments.push(dir);
|
||||
}
|
||||
}
|
||||
if (slug) segments.push(slug);
|
||||
@@ -218,6 +222,7 @@ function auditHtml(file, knownSlugs) {
|
||||
if (
|
||||
robots &&
|
||||
/noindex/i.test(robots) &&
|
||||
!isIndexingBlocked() &&
|
||||
!NOINDEX_ALLOWLIST.has(file.rel) &&
|
||||
!NOINDEX_ALLOWLIST.has(path.basename(file.rel))
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
export function resolveTranslation(langResources, key) {
|
||||
let namespace = 'common';
|
||||
let keyPath = key;
|
||||
const colonIndex = key.indexOf(':');
|
||||
if (colonIndex !== -1) {
|
||||
namespace = key.slice(0, colonIndex);
|
||||
keyPath = key.slice(colonIndex + 1);
|
||||
}
|
||||
let node = langResources[namespace];
|
||||
for (const segment of keyPath.split('.')) {
|
||||
if (node == null || typeof node !== 'object') return null;
|
||||
node = node[segment];
|
||||
}
|
||||
return typeof node === 'string' && node ? node : null;
|
||||
}
|
||||
|
||||
export function applyServerTranslations(document, langResources) {
|
||||
document.querySelectorAll('[data-i18n]').forEach((element) => {
|
||||
const key = element.getAttribute('data-i18n');
|
||||
if (!key) return;
|
||||
const translation = resolveTranslation(langResources, key);
|
||||
if (translation) element.textContent = translation;
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-i18n-placeholder]').forEach((element) => {
|
||||
const key = element.getAttribute('data-i18n-placeholder');
|
||||
if (!key) return;
|
||||
const translation = resolveTranslation(langResources, key);
|
||||
if (translation) element.setAttribute('placeholder', translation);
|
||||
});
|
||||
|
||||
document.querySelectorAll('[data-i18n-title]').forEach((element) => {
|
||||
const key = element.getAttribute('data-i18n-title');
|
||||
if (!key) return;
|
||||
const translation = resolveTranslation(langResources, key);
|
||||
if (translation) element.setAttribute('title', translation);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { validateTranslationPair } from './content-segments.mjs';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const SOURCE_FILE = path.join(ROOT, 'translations/source.json');
|
||||
const LOCALES_DIR = path.join(ROOT, 'public/locales');
|
||||
|
||||
const SCRIPT_RANGES = {
|
||||
ar: /[-ۿ]/,
|
||||
fa: /[-ۿ]/,
|
||||
be: /[Ѐ-ӿ]/,
|
||||
ru: /[Ѐ-ӿ]/,
|
||||
uk: /[Ѐ-ӿ]/,
|
||||
ja: /[-ヿ一-鿿]/,
|
||||
zh: /[一-鿿]/,
|
||||
'zh-TW': /[一-鿿]/,
|
||||
ko: /[가-]/,
|
||||
};
|
||||
|
||||
const ENGLISH_MARKERS = new Set([
|
||||
'the',
|
||||
'and',
|
||||
'your',
|
||||
'with',
|
||||
'you',
|
||||
'this',
|
||||
'that',
|
||||
'from',
|
||||
'are',
|
||||
'will',
|
||||
'how',
|
||||
'what',
|
||||
'when',
|
||||
'can',
|
||||
]);
|
||||
|
||||
function readJson(file, fallback) {
|
||||
return fs.existsSync(file)
|
||||
? JSON.parse(fs.readFileSync(file, 'utf-8'))
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function writeJson(file, data) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify(data, null, 2) + '\n');
|
||||
}
|
||||
|
||||
function contentFile(lang) {
|
||||
return path.join(ROOT, 'translations', lang, 'content.json');
|
||||
}
|
||||
|
||||
function languages() {
|
||||
return fs
|
||||
.readdirSync(LOCALES_DIR)
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry !== 'en' &&
|
||||
fs.statSync(path.join(LOCALES_DIR, entry)).isDirectory()
|
||||
)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function plainText(markup) {
|
||||
return markup.replace(/<\/?\d+\/?>/g, ' ');
|
||||
}
|
||||
|
||||
export function reviewTranslation(lang, source, translation) {
|
||||
const problem = validateTranslationPair(source, translation);
|
||||
if (problem) return { level: 'error', reason: problem };
|
||||
if (/[—]/.test(translation)) {
|
||||
return { level: 'error', reason: 'contains an em dash' };
|
||||
}
|
||||
const sourceText = plainText(source);
|
||||
const text = plainText(translation);
|
||||
const sourceWords = sourceText.split(/\s+/).filter(Boolean);
|
||||
if (
|
||||
translation.trim() === source.trim() &&
|
||||
(sourceWords.length < 6 || !/[a-z]{3,}/.test(sourceText))
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const range = SCRIPT_RANGES[lang];
|
||||
if (range && sourceWords.length >= 3 && !range.test(text)) {
|
||||
return { level: 'error', reason: 'not in the target script' };
|
||||
}
|
||||
const tokens = text
|
||||
.toLowerCase()
|
||||
.split(/[^\p{L}]+/u)
|
||||
.filter(Boolean);
|
||||
if (!range && tokens.length >= 6) {
|
||||
const hits = tokens.filter((token) => ENGLISH_MARKERS.has(token)).length;
|
||||
if (hits >= 2 && hits / tokens.length >= 0.15) {
|
||||
return { level: 'error', reason: 'looks like English' };
|
||||
}
|
||||
}
|
||||
const ratio = text.length / Math.max(1, sourceText.length);
|
||||
if (sourceWords.length >= 6 && (ratio > 3.5 || ratio < 0.2)) {
|
||||
return { level: 'warning', reason: `length ratio ${ratio.toFixed(2)}` };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function chunkCommand(workDir, wordsPerChunk, lang) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const done = lang ? readJson(contentFile(lang), {}) : {};
|
||||
const ids = Object.keys(source)
|
||||
.filter((id) => !done[id])
|
||||
.sort((a, b) => {
|
||||
const pa = source[a].pages[0] || '';
|
||||
const pb = source[b].pages[0] || '';
|
||||
return pa === pb ? a.localeCompare(b) : pa.localeCompare(pb);
|
||||
});
|
||||
const inDir = path.join(workDir, 'in');
|
||||
fs.mkdirSync(inDir, { recursive: true });
|
||||
let chunk = {};
|
||||
let words = 0;
|
||||
let index = 0;
|
||||
const flush = () => {
|
||||
if (Object.keys(chunk).length === 0) return;
|
||||
index += 1;
|
||||
writeJson(
|
||||
path.join(inDir, `chunk-${String(index).padStart(3, '0')}.json`),
|
||||
chunk
|
||||
);
|
||||
chunk = {};
|
||||
words = 0;
|
||||
};
|
||||
for (const id of ids) {
|
||||
const entry = source[id];
|
||||
chunk[id] = { text: entry.text, tags: entry.tags, page: entry.pages[0] };
|
||||
words += entry.text.split(/\s+/).length;
|
||||
if (words >= wordsPerChunk) flush();
|
||||
}
|
||||
flush();
|
||||
console.log(`Wrote ${index} chunks for ${ids.length} segments to ${inDir}`);
|
||||
}
|
||||
|
||||
function mergeCommand(lang, outDir, replace) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const target = contentFile(lang);
|
||||
const existing = readJson(target, {});
|
||||
const accepted = {};
|
||||
const rejected = [];
|
||||
const warnings = [];
|
||||
const files = fs.existsSync(outDir)
|
||||
? fs
|
||||
.readdirSync(outDir)
|
||||
.filter((f) => /^chunk-\d+\.json$/.test(f))
|
||||
.sort()
|
||||
: [];
|
||||
for (const file of files) {
|
||||
const data = readJson(path.join(outDir, file), {});
|
||||
for (const [id, translation] of Object.entries(data)) {
|
||||
if (!source[id]) {
|
||||
rejected.push({ id, reason: 'unknown id' });
|
||||
continue;
|
||||
}
|
||||
const review = reviewTranslation(lang, source[id].text, translation);
|
||||
if (review && review.level === 'error') {
|
||||
rejected.push({ id, reason: review.reason });
|
||||
continue;
|
||||
}
|
||||
if (review) warnings.push({ id, reason: review.reason });
|
||||
if (existing[id] && !replace) continue;
|
||||
accepted[id] = translation;
|
||||
}
|
||||
}
|
||||
const merged = { ...existing, ...accepted };
|
||||
const sorted = {};
|
||||
for (const id of Object.keys(merged).sort()) sorted[id] = merged[id];
|
||||
writeJson(target, sorted);
|
||||
writeJson(path.join(outDir, 'rejects.json'), rejected);
|
||||
const missing = Object.keys(source).filter((id) => !sorted[id]);
|
||||
console.log(
|
||||
`${lang}: accepted ${Object.keys(accepted).length}, rejected ${rejected.length}, warnings ${warnings.length}, still missing ${missing.length}`
|
||||
);
|
||||
}
|
||||
|
||||
function validateCommand(lang) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const content = readJson(contentFile(lang), {});
|
||||
const errors = [];
|
||||
const warnings = [];
|
||||
for (const [id, translation] of Object.entries(content)) {
|
||||
if (!source[id]) {
|
||||
warnings.push({ id, reason: 'stale id (run prune to remove)' });
|
||||
continue;
|
||||
}
|
||||
const review = reviewTranslation(lang, source[id].text, translation);
|
||||
if (review)
|
||||
(review.level === 'error' ? errors : warnings).push({
|
||||
id,
|
||||
reason: review.reason,
|
||||
});
|
||||
}
|
||||
const missing = Object.keys(source).filter((id) => !content[id]);
|
||||
const translated = Object.keys(content).filter((id) => source[id]).length;
|
||||
console.log(
|
||||
`${lang}: ${translated}/${Object.keys(source).length} translated, ${errors.length} errors, ${warnings.length} warnings, ${missing.length} missing`
|
||||
);
|
||||
return { errors, warnings, missing };
|
||||
}
|
||||
|
||||
function pruneCommand(lang) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const content = readJson(contentFile(lang), {});
|
||||
const kept = {};
|
||||
for (const id of Object.keys(content).sort()) {
|
||||
if (source[id]) kept[id] = content[id];
|
||||
}
|
||||
writeJson(contentFile(lang), kept);
|
||||
console.log(
|
||||
`${lang}: removed ${Object.keys(content).length - Object.keys(kept).length} stale ids`
|
||||
);
|
||||
}
|
||||
|
||||
function statusCommand() {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const total = Object.keys(source).length;
|
||||
for (const lang of languages()) {
|
||||
const content = readJson(contentFile(lang), {});
|
||||
const done = Object.keys(content).filter((id) => source[id]).length;
|
||||
console.log(
|
||||
`${lang.padEnd(6)} ${done}/${total} (${Math.round((done / total) * 100)}%)`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
const [command, ...args] = process.argv.slice(2);
|
||||
const langIndex = args.indexOf('--lang');
|
||||
const langOption = langIndex >= 0 ? args[langIndex + 1] : undefined;
|
||||
const flags = new Set(args.filter((arg) => arg.startsWith('--')));
|
||||
const values = args.filter(
|
||||
(arg, index) =>
|
||||
!arg.startsWith('--') && (langIndex < 0 || index !== langIndex + 1)
|
||||
);
|
||||
if (command === 'chunk') {
|
||||
chunkCommand(values[0], Number(values[1] || 900), langOption);
|
||||
} else if (command === 'merge') {
|
||||
mergeCommand(values[0], values[1], flags.has('--replace'));
|
||||
} else if (command === 'prune') pruneCommand(values[0]);
|
||||
else if (command === 'validate') {
|
||||
const result = validateCommand(args[0]);
|
||||
if (args[1] === '--list') {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
errors: result.errors.slice(0, 50),
|
||||
warnings: result.warnings.slice(0, 20),
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
}
|
||||
process.exitCode = result.errors.length > 0 ? 1 : 0;
|
||||
} else if (command === 'status') statusCommand();
|
||||
else {
|
||||
console.error(
|
||||
'Usage: translations.mjs chunk <workDir> [words] [--lang <lang>] | merge <lang> <outDir> [--replace] | validate <lang> [--list] | prune <lang> | status'
|
||||
);
|
||||
process.exitCode = 2;
|
||||
}
|
||||
}
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) main();
|
||||
@@ -0,0 +1,200 @@
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { reviewTranslation } from './translations.mjs';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const SOURCE_FILE = path.join(ROOT, 'translations/ui-source.json');
|
||||
const LOCALES_DIR = path.join(ROOT, 'public/locales');
|
||||
|
||||
function readJson(file, fallback) {
|
||||
return fs.existsSync(file)
|
||||
? JSON.parse(fs.readFileSync(file, 'utf-8'))
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function writeJson(file, data) {
|
||||
fs.mkdirSync(path.dirname(file), { recursive: true });
|
||||
fs.writeFileSync(file, JSON.stringify(data, null, 2) + '\n');
|
||||
}
|
||||
|
||||
function targetFile(lang) {
|
||||
return path.join(LOCALES_DIR, lang, 'ui.json');
|
||||
}
|
||||
|
||||
function idOf(text) {
|
||||
return createHash('sha1').update(text).digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
function placeholderSignature(text) {
|
||||
return (text.match(/\{\{\w+\}\}/g) || []).sort().join(',');
|
||||
}
|
||||
|
||||
function languages() {
|
||||
return fs
|
||||
.readdirSync(LOCALES_DIR)
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry !== 'en' &&
|
||||
fs.statSync(path.join(LOCALES_DIR, entry)).isDirectory()
|
||||
)
|
||||
.sort();
|
||||
}
|
||||
|
||||
function needsTranslation(text) {
|
||||
return /\p{L}{2}/u.test(text.replace(/\{\{\w+\}\}/g, ' '));
|
||||
}
|
||||
|
||||
function review(lang, source, translation) {
|
||||
if (!needsTranslation(source) && translation === source) return null;
|
||||
if (typeof translation !== 'string' || translation.trim() === '') {
|
||||
return { level: 'error', reason: 'empty' };
|
||||
}
|
||||
if (placeholderSignature(source) !== placeholderSignature(translation)) {
|
||||
return { level: 'error', reason: 'placeholders differ' };
|
||||
}
|
||||
if (/<\/?[a-z][^>]*>/i.test(translation)) {
|
||||
return { level: 'error', reason: 'contains HTML' };
|
||||
}
|
||||
const stripped = (text) => text.replace(/\{\{\w+\}\}/g, ' ');
|
||||
return reviewTranslation(lang, stripped(source), stripped(translation));
|
||||
}
|
||||
|
||||
function chunkCommand(workDir, wordsPerChunk) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const inDir = path.join(workDir, 'in');
|
||||
fs.mkdirSync(inDir, { recursive: true });
|
||||
let chunk = {};
|
||||
let words = 0;
|
||||
let index = 0;
|
||||
const flush = () => {
|
||||
if (Object.keys(chunk).length === 0) return;
|
||||
index += 1;
|
||||
writeJson(
|
||||
path.join(inDir, `chunk-${String(index).padStart(3, '0')}.json`),
|
||||
chunk
|
||||
);
|
||||
chunk = {};
|
||||
words = 0;
|
||||
};
|
||||
for (const text of Object.keys(source).sort()) {
|
||||
chunk[idOf(text)] = {
|
||||
text,
|
||||
tags: ['ui'],
|
||||
page: source[text].files[0] || '',
|
||||
};
|
||||
words += text.split(/\s+/).length;
|
||||
if (words >= wordsPerChunk) flush();
|
||||
}
|
||||
flush();
|
||||
console.log(
|
||||
`Wrote ${index} chunks for ${Object.keys(source).length} strings`
|
||||
);
|
||||
}
|
||||
|
||||
function mergeCommand(lang, outDir, replace) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const byId = new Map(Object.keys(source).map((text) => [idOf(text), text]));
|
||||
const existing = readJson(targetFile(lang), {});
|
||||
const accepted = {};
|
||||
const rejected = [];
|
||||
const files = fs.existsSync(outDir)
|
||||
? fs
|
||||
.readdirSync(outDir)
|
||||
.filter((file) => /^chunk-\d+\.json$/.test(file))
|
||||
.sort()
|
||||
: [];
|
||||
for (const file of files) {
|
||||
const data = readJson(path.join(outDir, file), {});
|
||||
for (const [id, translation] of Object.entries(data)) {
|
||||
const text = byId.get(id);
|
||||
if (!text) {
|
||||
rejected.push({ id, reason: 'unknown id' });
|
||||
continue;
|
||||
}
|
||||
const result = review(lang, text, translation);
|
||||
if (result && result.level === 'error') {
|
||||
rejected.push({ id, reason: result.reason });
|
||||
continue;
|
||||
}
|
||||
if (existing[text] && !replace) continue;
|
||||
accepted[text] = translation;
|
||||
}
|
||||
}
|
||||
const merged = { ...existing, ...accepted };
|
||||
for (const text of Object.keys(source)) {
|
||||
if (!needsTranslation(text) && !merged[text]) merged[text] = text;
|
||||
}
|
||||
const sorted = {};
|
||||
for (const text of Object.keys(merged).sort()) sorted[text] = merged[text];
|
||||
writeJson(targetFile(lang), sorted);
|
||||
writeJson(path.join(outDir, 'rejects.json'), rejected);
|
||||
const missing = Object.keys(source).filter((text) => !sorted[text]);
|
||||
console.log(
|
||||
`${lang}: accepted ${Object.keys(accepted).length}, rejected ${rejected.length}, still missing ${missing.length}`
|
||||
);
|
||||
}
|
||||
|
||||
function validateCommand(lang) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const content = readJson(targetFile(lang), {});
|
||||
const errors = [];
|
||||
const stale = [];
|
||||
for (const [text, translation] of Object.entries(content)) {
|
||||
if (!source[text]) {
|
||||
stale.push(text);
|
||||
continue;
|
||||
}
|
||||
const result = review(lang, text, translation);
|
||||
if (result && result.level === 'error') {
|
||||
errors.push({ text, reason: result.reason });
|
||||
}
|
||||
}
|
||||
const missing = Object.keys(source).filter((text) => !content[text]);
|
||||
const translated = Object.keys(content).length - stale.length;
|
||||
console.log(
|
||||
`${lang}: ${translated}/${Object.keys(source).length} translated, ${errors.length} errors, ${missing.length} missing${stale.length ? `, ${stale.length} stale (run prune to remove)` : ''}`
|
||||
);
|
||||
return { errors, missing, stale };
|
||||
}
|
||||
|
||||
function pruneCommand(lang) {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const content = readJson(targetFile(lang), {});
|
||||
const kept = {};
|
||||
for (const text of Object.keys(content).sort()) {
|
||||
if (source[text]) kept[text] = content[text];
|
||||
}
|
||||
writeJson(targetFile(lang), kept);
|
||||
console.log(
|
||||
`${lang}: removed ${Object.keys(content).length - Object.keys(kept).length} stale keys`
|
||||
);
|
||||
}
|
||||
|
||||
function statusCommand() {
|
||||
const source = readJson(SOURCE_FILE, {});
|
||||
const total = Object.keys(source).length;
|
||||
for (const lang of languages()) {
|
||||
const content = readJson(targetFile(lang), {});
|
||||
const done = Object.keys(content).filter((text) => source[text]).length;
|
||||
console.log(`${lang.padEnd(6)} ${done}/${total}`);
|
||||
}
|
||||
}
|
||||
|
||||
const [command, ...args] = process.argv.slice(2);
|
||||
if (command === 'chunk') chunkCommand(args[0], Number(args[1] || 800));
|
||||
else if (command === 'merge')
|
||||
mergeCommand(args[0], args[1], args.includes('--replace'));
|
||||
else if (command === 'validate') {
|
||||
const result = validateCommand(args[0]);
|
||||
process.exitCode = result.errors.length > 0 ? 1 : 0;
|
||||
} else if (command === 'prune') pruneCommand(args[0]);
|
||||
else if (command === 'status') statusCommand();
|
||||
else {
|
||||
console.error(
|
||||
'Usage: ui-translations.mjs chunk <workDir> [words] | merge <lang> <outDir> [--replace] | validate <lang> | prune <lang> | status'
|
||||
);
|
||||
process.exitCode = 2;
|
||||
}
|
||||
Reference in New Issue
Block a user