mirror of
https://github.com/alam00000/bentopdf.git
synced 2026-11-09 00:57:15 +01:00
feat: translate all pages, UI messages and docs and fix SEO redirects and indexing
This commit is contained in:
alam00000 committed
1 parent
5e019de5ae
commit
4edb1718cc
3344 files changed
+415643
-4759
No files matched your search
@@ -0,0 +1,571 @@
|
||||
import { createHash } from 'crypto';
|
||||
import { serializeJsonLd } from './json-ld.mjs';
|
||||
|
||||
const INLINE_TAGS = new Set([
|
||||
'a',
|
||||
'abbr',
|
||||
'b',
|
||||
'bdi',
|
||||
'bdo',
|
||||
'cite',
|
||||
'data',
|
||||
'del',
|
||||
'dfn',
|
||||
'em',
|
||||
'i',
|
||||
'ins',
|
||||
'mark',
|
||||
'q',
|
||||
's',
|
||||
'small',
|
||||
'span',
|
||||
'strong',
|
||||
'sub',
|
||||
'sup',
|
||||
'time',
|
||||
'u',
|
||||
]);
|
||||
|
||||
const ALWAYS_OPAQUE_TAGS = new Set([
|
||||
'audio',
|
||||
'br',
|
||||
'canvas',
|
||||
'code',
|
||||
'iframe',
|
||||
'img',
|
||||
'input',
|
||||
'kbd',
|
||||
'math',
|
||||
'samp',
|
||||
'svg',
|
||||
'var',
|
||||
'video',
|
||||
'wbr',
|
||||
]);
|
||||
|
||||
const SKIPPED_TAGS = new Set([
|
||||
'head',
|
||||
'noscript',
|
||||
'pre',
|
||||
'script',
|
||||
'style',
|
||||
'template',
|
||||
'textarea',
|
||||
]);
|
||||
|
||||
const TRANSLATABLE_ATTRIBUTES = ['alt', 'title', 'placeholder', 'aria-label'];
|
||||
|
||||
const META_SELECTORS = [
|
||||
'meta[name="description"]',
|
||||
'meta[property="og:title"]',
|
||||
'meta[property="og:description"]',
|
||||
'meta[name="twitter:title"]',
|
||||
'meta[name="twitter:description"]',
|
||||
];
|
||||
|
||||
const PROTECTED_TERMS = [
|
||||
'BentoPDF',
|
||||
'Hyper Compress',
|
||||
'Kura',
|
||||
'GitHub',
|
||||
'PyMuPDF',
|
||||
'Ghostscript',
|
||||
'CoherentPDF',
|
||||
'LibreOffice',
|
||||
'Tesseract',
|
||||
'DigitalOcean',
|
||||
'Fubarpie',
|
||||
'Trustpilot',
|
||||
'LinkedIn',
|
||||
'Discord',
|
||||
'WebAssembly',
|
||||
'PDF/A',
|
||||
'PDF/UA',
|
||||
'PDF/X',
|
||||
'PDF',
|
||||
];
|
||||
|
||||
const PROTECTED_PATTERN = new RegExp(
|
||||
PROTECTED_TERMS.map((term) =>
|
||||
term.replace(/[.*+?^${}()|[\]\\/]/g, '\\$&')
|
||||
).join('|'),
|
||||
'gi'
|
||||
);
|
||||
|
||||
const LETTERS = /\p{L}{2}/u;
|
||||
const PLACEHOLDER_SOURCE = '<(\\/?)(\\d+)(\\/?)>';
|
||||
const SPACES = /[ \t\r\n\f]+/g;
|
||||
|
||||
function placeholders(text) {
|
||||
return Array.from(text.matchAll(new RegExp(PLACEHOLDER_SOURCE, 'g')));
|
||||
}
|
||||
|
||||
function hasPlaceholder(text) {
|
||||
return new RegExp(PLACEHOLDER_SOURCE).test(text);
|
||||
}
|
||||
|
||||
export function segmentId(markup) {
|
||||
return createHash('sha1').update(markup).digest('hex').slice(0, 12);
|
||||
}
|
||||
|
||||
function escapeText(text) {
|
||||
return text
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
function decodeText(text) {
|
||||
return text
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/&/g, '&');
|
||||
}
|
||||
|
||||
function isTranslatable(markup) {
|
||||
const text = markup
|
||||
.replace(new RegExp(PLACEHOLDER_SOURCE, 'g'), ' ')
|
||||
.replace(PROTECTED_PATTERN, ' ');
|
||||
return LETTERS.test(text);
|
||||
}
|
||||
|
||||
function normalize(text) {
|
||||
return text.replace(SPACES, ' ').trim();
|
||||
}
|
||||
|
||||
function tagOf(element) {
|
||||
return element.tagName.toLowerCase();
|
||||
}
|
||||
|
||||
function isInlineLike(element) {
|
||||
const tag = tagOf(element);
|
||||
if (ALWAYS_OPAQUE_TAGS.has(tag)) return true;
|
||||
if (!INLINE_TAGS.has(tag)) return false;
|
||||
if (element.hasAttribute('data-i18n')) return true;
|
||||
return Array.from(element.children).every(isInlineLike);
|
||||
}
|
||||
|
||||
function isSkippedBlock(element) {
|
||||
return (
|
||||
SKIPPED_TAGS.has(tagOf(element)) ||
|
||||
element.hasAttribute('data-i18n') ||
|
||||
element.getAttribute('translate') === 'no' ||
|
||||
element.hasAttribute('data-no-i18n') ||
|
||||
element.hasAttribute('contenteditable')
|
||||
);
|
||||
}
|
||||
|
||||
function isOpaque(element) {
|
||||
return (
|
||||
ALWAYS_OPAQUE_TAGS.has(tagOf(element)) ||
|
||||
element.hasAttribute('data-i18n') ||
|
||||
element.getAttribute('translate') === 'no' ||
|
||||
!LETTERS.test(element.textContent || '')
|
||||
);
|
||||
}
|
||||
|
||||
function serialize(nodes, state) {
|
||||
let out = '';
|
||||
for (const node of nodes) {
|
||||
if (node.nodeType === 3) {
|
||||
out += escapeText(node.data);
|
||||
} else if (node.nodeType === 1) {
|
||||
state.count += 1;
|
||||
const index = state.count;
|
||||
state.elements[index] = node;
|
||||
if (isOpaque(node)) {
|
||||
out += `<${index}/>`;
|
||||
} else {
|
||||
out += `<${index}>${serialize(Array.from(node.childNodes), state)}</${index}>`;
|
||||
}
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function parseTranslation(text, elements) {
|
||||
const root = { children: [] };
|
||||
const stack = [root];
|
||||
const seen = new Set();
|
||||
let last = 0;
|
||||
const pushText = (piece) => {
|
||||
if (!piece) return true;
|
||||
if (/[<>]/.test(piece)) return false;
|
||||
stack[stack.length - 1].children.push({ text: decodeText(piece) });
|
||||
return true;
|
||||
};
|
||||
for (const match of placeholders(text)) {
|
||||
if (!pushText(text.slice(last, match.index))) return null;
|
||||
last = match.index + match[0].length;
|
||||
const closing = match[1] === '/';
|
||||
const selfClosing = match[3] === '/';
|
||||
const index = Number(match[2]);
|
||||
const source = elements[index];
|
||||
if (!source || (closing && selfClosing)) return null;
|
||||
if (selfClosing) {
|
||||
if (!isOpaque(source) || seen.has(index)) return null;
|
||||
seen.add(index);
|
||||
stack[stack.length - 1].children.push({ element: index });
|
||||
} else if (!closing) {
|
||||
if (isOpaque(source) || seen.has(index)) return null;
|
||||
seen.add(index);
|
||||
const node = { element: index, children: [] };
|
||||
stack[stack.length - 1].children.push(node);
|
||||
stack.push(node);
|
||||
} else {
|
||||
const top = stack[stack.length - 1];
|
||||
if (top.element !== index) return null;
|
||||
stack.pop();
|
||||
}
|
||||
}
|
||||
if (!pushText(text.slice(last))) return null;
|
||||
if (stack.length !== 1) return null;
|
||||
if (seen.size !== Object.keys(elements).length) return null;
|
||||
return root.children;
|
||||
}
|
||||
|
||||
function buildFragment(document, children, elements) {
|
||||
const fragment = document.createDocumentFragment();
|
||||
for (const child of children) {
|
||||
if ('text' in child) {
|
||||
fragment.appendChild(document.createTextNode(child.text));
|
||||
} else if (child.children) {
|
||||
const clone = elements[child.element].cloneNode(false);
|
||||
clone.appendChild(buildFragment(document, child.children, elements));
|
||||
fragment.appendChild(clone);
|
||||
} else {
|
||||
fragment.appendChild(elements[child.element].cloneNode(true));
|
||||
}
|
||||
}
|
||||
return fragment;
|
||||
}
|
||||
|
||||
function significantNodes(nodes) {
|
||||
return nodes.filter(
|
||||
(node) => node.nodeType !== 3 || normalize(node.data) !== ''
|
||||
);
|
||||
}
|
||||
|
||||
function handleSegment(markup, tag, context, apply) {
|
||||
const { options, stats } = context;
|
||||
const id = segmentId(markup);
|
||||
stats.found += 1;
|
||||
if (options.record) options.record(id, markup, tag);
|
||||
if (!options.translate) return;
|
||||
const translation = options.translate(id);
|
||||
if (!translation) return;
|
||||
if (apply(translation)) stats.translated += 1;
|
||||
else stats.invalid.add(id);
|
||||
}
|
||||
|
||||
function handleAttributes(element, context) {
|
||||
for (const name of TRANSLATABLE_ATTRIBUTES) {
|
||||
if (!element.hasAttribute(name)) continue;
|
||||
if (element.hasAttribute(`data-i18n-${name}`)) continue;
|
||||
const value = normalize(element.getAttribute(name));
|
||||
if (!isTranslatable(escapeText(value))) continue;
|
||||
handleSegment(escapeText(value), tagOf(element), context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
element.setAttribute(name, decodeText(translation));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function handleRun(nodes, tag, context) {
|
||||
const significant = significantNodes(nodes);
|
||||
if (significant.length === 0) return;
|
||||
for (const node of nodes) {
|
||||
if (node.nodeType !== 1) continue;
|
||||
handleAttributes(node, context);
|
||||
node
|
||||
.querySelectorAll('*')
|
||||
.forEach((child) => handleAttributes(child, context));
|
||||
}
|
||||
if (
|
||||
significant.length === 1 &&
|
||||
significant[0].nodeType === 1 &&
|
||||
!isOpaque(significant[0])
|
||||
) {
|
||||
handleRun(Array.from(significant[0].childNodes), tag, context);
|
||||
return;
|
||||
}
|
||||
const state = { count: 0, elements: {} };
|
||||
const markup = normalize(serialize(nodes, state));
|
||||
if (!isTranslatable(markup)) return;
|
||||
handleSegment(markup, tag, context, (translation) => {
|
||||
const children = parseTranslation(translation, state.elements);
|
||||
if (!children) return false;
|
||||
const document = significant[0].ownerDocument;
|
||||
const fragment = buildFragment(document, children, state.elements);
|
||||
const anchor = nodes[0];
|
||||
anchor.parentNode.insertBefore(fragment, anchor);
|
||||
nodes.forEach((node) => node.parentNode.removeChild(node));
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function visitChildren(parent, context) {
|
||||
const tag = tagOf(parent);
|
||||
let run = [];
|
||||
const flush = () => {
|
||||
if (run.length > 0) handleRun(run, tag, context);
|
||||
run = [];
|
||||
};
|
||||
for (const node of Array.from(parent.childNodes)) {
|
||||
if (node.nodeType === 3) {
|
||||
run.push(node);
|
||||
} else if (node.nodeType === 1 && isInlineLike(node)) {
|
||||
run.push(node);
|
||||
} else {
|
||||
flush();
|
||||
if (node.nodeType === 1 && !isSkippedBlock(node)) {
|
||||
handleAttributes(node, context);
|
||||
visitChildren(node, context);
|
||||
}
|
||||
}
|
||||
}
|
||||
flush();
|
||||
}
|
||||
|
||||
function processMeta(document, context) {
|
||||
const title = document.querySelector('title');
|
||||
if (title) {
|
||||
const value = normalize(title.textContent || '');
|
||||
if (isTranslatable(escapeText(value))) {
|
||||
handleSegment(escapeText(value), 'title', context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
title.textContent = decodeText(translation);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const selector of META_SELECTORS) {
|
||||
document.querySelectorAll(selector).forEach((meta) => {
|
||||
const value = normalize(meta.getAttribute('content') || '');
|
||||
if (!isTranslatable(escapeText(value))) return;
|
||||
handleSegment(escapeText(value), 'meta', context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
meta.setAttribute('content', decodeText(translation));
|
||||
return true;
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const STRUCTURED_DATA_TEXT_KEYS = new Set([
|
||||
'name',
|
||||
'description',
|
||||
'text',
|
||||
'headline',
|
||||
]);
|
||||
|
||||
const STRUCTURED_DATA_SKIPPED_TYPES = new Set([
|
||||
'BreadcrumbList',
|
||||
'FAQPage',
|
||||
'ImageObject',
|
||||
'ListItem',
|
||||
'Offer',
|
||||
'Organization',
|
||||
'Person',
|
||||
'Question',
|
||||
'WebSite',
|
||||
]);
|
||||
|
||||
function walkStructuredData(value, handle) {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((item) => walkStructuredData(item, handle));
|
||||
return;
|
||||
}
|
||||
if (!value || typeof value !== 'object') return;
|
||||
if (STRUCTURED_DATA_SKIPPED_TYPES.has(value['@type'])) return;
|
||||
for (const [key, child] of Object.entries(value)) {
|
||||
if (typeof child === 'string') {
|
||||
if (STRUCTURED_DATA_TEXT_KEYS.has(key)) handle(value, key);
|
||||
} else if (key === 'featureList' && Array.isArray(child)) {
|
||||
child.forEach((item, index) => {
|
||||
if (typeof item === 'string') handle(child, index);
|
||||
});
|
||||
} else {
|
||||
walkStructuredData(child, handle);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function processStructuredData(document, context) {
|
||||
document
|
||||
.querySelectorAll('script[type="application/ld+json"]')
|
||||
.forEach((script) => {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(script.textContent || '');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
let changed = false;
|
||||
walkStructuredData(data, (container, key) => {
|
||||
const value = normalize(container[key]);
|
||||
if (!isTranslatable(escapeText(value))) return;
|
||||
handleSegment(escapeText(value), 'ld', context, (translation) => {
|
||||
if (hasPlaceholder(translation)) return false;
|
||||
container[key] = decodeText(translation);
|
||||
changed = true;
|
||||
return true;
|
||||
});
|
||||
});
|
||||
if (changed) script.textContent = serializeJsonLd(data);
|
||||
});
|
||||
}
|
||||
|
||||
const BLOCK_TAGS = new Set([
|
||||
'p',
|
||||
'div',
|
||||
'ul',
|
||||
'ol',
|
||||
'li',
|
||||
'table',
|
||||
'tr',
|
||||
'td',
|
||||
'th',
|
||||
'h1',
|
||||
'h2',
|
||||
'h3',
|
||||
'h4',
|
||||
'h5',
|
||||
'h6',
|
||||
'blockquote',
|
||||
'section',
|
||||
]);
|
||||
|
||||
function blockText(node) {
|
||||
if (node.nodeType !== 1) return node.textContent || '';
|
||||
const text = Array.from(node.childNodes).map(blockText).join('');
|
||||
return BLOCK_TAGS.has(tagOf(node)) ? ` ${text} ` : text;
|
||||
}
|
||||
|
||||
function rebuildFaqSchema(document) {
|
||||
const items = [];
|
||||
for (const details of document.querySelectorAll('details.faq-d')) {
|
||||
const summary = details.querySelector('summary');
|
||||
if (!summary) continue;
|
||||
const question = normalize(summary.textContent || '');
|
||||
const answerText = normalize(
|
||||
Array.from(details.childNodes)
|
||||
.filter((child) => child !== summary)
|
||||
.map(blockText)
|
||||
.join(' ')
|
||||
);
|
||||
if (!question || !answerText) continue;
|
||||
items.push({
|
||||
'@type': 'Question',
|
||||
name: question,
|
||||
acceptedAnswer: { '@type': 'Answer', text: answerText },
|
||||
});
|
||||
}
|
||||
if (items.length === 0) return;
|
||||
document
|
||||
.querySelectorAll('script[type="application/ld+json"]')
|
||||
.forEach((script) => {
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(script.textContent || '');
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (data && data['@type'] === 'FAQPage') {
|
||||
script.textContent = serializeJsonLd({ ...data, mainEntity: items });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function processDocument(document, options) {
|
||||
const context = {
|
||||
options,
|
||||
stats: { found: 0, translated: 0, invalid: new Set() },
|
||||
};
|
||||
if (!options.skipMeta) processMeta(document, context);
|
||||
const roots = options.roots
|
||||
? options.roots(document)
|
||||
: [document.body].filter(Boolean);
|
||||
for (const root of roots) {
|
||||
if (!isSkippedBlock(root)) {
|
||||
handleAttributes(root, context);
|
||||
visitChildren(root, context);
|
||||
}
|
||||
}
|
||||
if (!options.skipMeta) {
|
||||
if (options.translate) rebuildFaqSchema(document);
|
||||
processStructuredData(document, context);
|
||||
}
|
||||
return {
|
||||
found: context.stats.found,
|
||||
translated: context.stats.translated,
|
||||
invalid: [...context.stats.invalid],
|
||||
};
|
||||
}
|
||||
|
||||
export const ENGLISH_ONLY_PAGES = new Set(['terms', 'privacy', 'licensing']);
|
||||
|
||||
export function processPage(document, pageName, options) {
|
||||
if (ENGLISH_ONLY_PAGES.has(pageName)) {
|
||||
return processDocument(document, {
|
||||
...options,
|
||||
skipMeta: true,
|
||||
roots: shellRoots,
|
||||
});
|
||||
}
|
||||
return processDocument(document, options);
|
||||
}
|
||||
|
||||
export function shellRoots(document) {
|
||||
return Array.from(document.querySelectorAll('header, nav, footer')).filter(
|
||||
(element) => !element.parentElement?.closest('header, nav, footer')
|
||||
);
|
||||
}
|
||||
|
||||
function placeholderParents(markup) {
|
||||
const stack = [];
|
||||
const parents = [];
|
||||
for (const match of placeholders(markup)) {
|
||||
const [, closing, index, selfClosing] = match;
|
||||
if (closing) {
|
||||
if (stack.pop() !== index) return null;
|
||||
} else {
|
||||
parents.push(`${index}:${stack.at(-1) ?? ''}`);
|
||||
if (!selfClosing) stack.push(index);
|
||||
}
|
||||
}
|
||||
return stack.length === 0 ? parents.sort().join(',') : null;
|
||||
}
|
||||
|
||||
function placeholderSignature(markup) {
|
||||
return placeholders(markup)
|
||||
.map((match) => `${match[1]}${match[2]}${match[3]}`)
|
||||
.sort()
|
||||
.join(',');
|
||||
}
|
||||
|
||||
export function validateTranslationPair(sourceMarkup, translation) {
|
||||
if (typeof translation !== 'string' || translation.trim() === '') {
|
||||
return 'empty';
|
||||
}
|
||||
if (
|
||||
placeholderSignature(sourceMarkup) !== placeholderSignature(translation)
|
||||
) {
|
||||
return 'placeholders differ';
|
||||
}
|
||||
const nesting = placeholderParents(translation);
|
||||
if (nesting === null || nesting !== placeholderParents(sourceMarkup)) {
|
||||
return 'placeholders nested differently';
|
||||
}
|
||||
const stripped = translation.replace(new RegExp(PLACEHOLDER_SOURCE, 'g'), '');
|
||||
if (/[<>]/.test(stripped)) return 'contains raw angle brackets';
|
||||
const unknownEntity = stripped.match(
|
||||
/&(?!(?:amp|lt|gt|quot|nbsp|#39);)[#\w]+;/
|
||||
);
|
||||
if (unknownEntity) return `unsupported entity ${unknownEntity[0]}`;
|
||||
return null;
|
||||
}
|
||||
Reference in new issue
Block a user