diff --git a/packages/compiler-cli/integrationtest/test/i18n_spec.ts b/packages/compiler-cli/integrationtest/test/i18n_spec.ts index 019b354ce9..63b914ff97 100644 --- a/packages/compiler-cli/integrationtest/test/i18n_spec.ts +++ b/packages/compiler-cli/integrationtest/test/i18n_spec.ts @@ -63,6 +63,32 @@ const EXPECTED_XLIFF = ` `; +const EXPECTED_XLIFF2 = ` + + + + + desc + meaning + + + translate me + + + + + Welcome + + + + + other-3rdP-component + + + + +`; + describe('template i18n extraction output', () => { const outDir = ''; const genDir = 'out'; @@ -81,6 +107,13 @@ describe('template i18n extraction output', () => { expect(xlf).toEqual(EXPECTED_XLIFF); }); + it('should extract i18n messages as xliff version 2.0', () => { + const xlfOutput = path.join(outDir, 'messages.xliff2.xlf'); + expect(fs.existsSync(xlfOutput)).toBeTruthy(); + const xlf = fs.readFileSync(xlfOutput, {encoding: 'utf-8'}); + expect(xlf).toEqual(EXPECTED_XLIFF2); + }); + it('should not emit js', () => { const genOutput = path.join(genDir, ''); expect(fs.existsSync(genOutput)).toBeFalsy(); diff --git a/packages/compiler-cli/src/extractor.ts b/packages/compiler-cli/src/extractor.ts index 1d929c8e7a..4b340f39dd 100644 --- a/packages/compiler-cli/src/extractor.ts +++ b/packages/compiler-cli/src/extractor.ts @@ -34,7 +34,7 @@ export class Extractor { const promiseBundle = this.extractBundle(); return promiseBundle.then(bundle => { - const content = this.serialize(bundle, ext); + const content = this.serialize(bundle, formatName); const dstFile = outFile || `messages.${ext}`; const dstPath = path.join(this.options.genDir, dstFile); this.host.writeFile(dstPath, content, false); @@ -48,14 +48,20 @@ export class Extractor { return this.ngExtractor.extract(files); } - serialize(bundle: compiler.MessageBundle, ext: string): string { + serialize(bundle: compiler.MessageBundle, formatName: string): string { + const format = formatName.toLowerCase(); let serializer: compiler.Serializer; - switch (ext) { + switch (format) { case 'xmb': serializer = new compiler.Xmb(); break; + case 'xliff2': + case 'xlf2': + serializer = new compiler.Xliff2(); + break; case 'xlf': + case 'xliff': default: serializer = new compiler.Xliff(); } @@ -66,10 +72,18 @@ export class Extractor { getExtension(formatName: string): string { const format = (formatName || 'xlf').toLowerCase(); - if (format === 'xmb') return 'xmb'; - if (format === 'xlf' || format === 'xlif' || format === 'xliff') return 'xlf'; + switch (format) { + case 'xmb': + return 'xmb'; + case 'xlf': + case 'xlif': + case 'xliff': + case 'xlf2': + case 'xliff2': + return 'xlf'; + } - throw new Error('Unsupported format "${formatName}"'); + throw new Error(`Unsupported format "${formatName}"`); } static create( diff --git a/packages/compiler/src/i18n/i18n_html_parser.ts b/packages/compiler/src/i18n/i18n_html_parser.ts index 3cb35db441..7a05ba42ce 100644 --- a/packages/compiler/src/i18n/i18n_html_parser.ts +++ b/packages/compiler/src/i18n/i18n_html_parser.ts @@ -13,6 +13,7 @@ import {ParseTreeResult} from '../ml_parser/parser'; import {mergeTranslations} from './extractor_merger'; import {Serializer} from './serializers/serializer'; import {Xliff} from './serializers/xliff'; +import {Xliff2} from './serializers/xliff2'; import {Xmb} from './serializers/xmb'; import {Xtb} from './serializers/xtb'; import {TranslationBundle} from './translation_bundle'; @@ -62,6 +63,9 @@ function createSerializer(format?: string): Serializer { return new Xmb(); case 'xtb': return new Xtb(); + case 'xliff2': + case 'xlf2': + return new Xliff2(); case 'xliff': case 'xlf': default: diff --git a/packages/compiler/src/i18n/index.ts b/packages/compiler/src/i18n/index.ts index eb05fb1c48..f47e971fd8 100644 --- a/packages/compiler/src/i18n/index.ts +++ b/packages/compiler/src/i18n/index.ts @@ -11,5 +11,6 @@ export {I18NHtmlParser} from './i18n_html_parser'; export {MessageBundle} from './message_bundle'; export {Serializer} from './serializers/serializer'; export {Xliff} from './serializers/xliff'; +export {Xliff2} from './serializers/xliff2'; export {Xmb} from './serializers/xmb'; export {Xtb} from './serializers/xtb'; diff --git a/packages/compiler/src/i18n/serializers/xliff2.ts b/packages/compiler/src/i18n/serializers/xliff2.ts new file mode 100644 index 0000000000..cb7fd960c8 --- /dev/null +++ b/packages/compiler/src/i18n/serializers/xliff2.ts @@ -0,0 +1,366 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import * as ml from '../../ml_parser/ast'; +import {XmlParser} from '../../ml_parser/xml_parser'; +import {decimalDigest} from '../digest'; +import * as i18n from '../i18n_ast'; +import {I18nError} from '../parse_util'; + +import {Serializer} from './serializer'; +import * as xml from './xml_helper'; + +const _VERSION = '2.0'; +const _XMLNS = 'urn:oasis:names:tc:xliff:document:2.0'; +// TODO(vicb): make this a param (s/_/-/) +const _DEFAULT_SOURCE_LANG = 'en'; +const _PLACEHOLDER_TAG = 'ph'; +const _PLACEHOLDER_SPANNING_TAG = 'pc'; + +const _XLIFF_TAG = 'xliff'; +const _SOURCE_TAG = 'source'; +const _TARGET_TAG = 'target'; +const _UNIT_TAG = 'unit'; + +// http://docs.oasis-open.org/xliff/xliff-core/v2.0/os/xliff-core-v2.0-os.html +export class Xliff2 extends Serializer { + write(messages: i18n.Message[], locale: string|null): string { + const visitor = new _WriteVisitor(); + const units: xml.Node[] = []; + + messages.forEach(message => { + const unit = new xml.Tag(_UNIT_TAG, {id: message.id}); + + if (message.description || message.meaning) { + const notes = new xml.Tag('notes'); + if (message.description) { + notes.children.push( + new xml.CR(8), + new xml.Tag('note', {category: 'description'}, [new xml.Text(message.description)])); + } + + if (message.meaning) { + notes.children.push( + new xml.CR(8), + new xml.Tag('note', {category: 'meaning'}, [new xml.Text(message.meaning)])); + } + + notes.children.push(new xml.CR(6)); + unit.children.push(new xml.CR(6), notes); + } + + const segment = new xml.Tag('segment'); + + segment.children.push( + new xml.CR(8), new xml.Tag(_SOURCE_TAG, {}, visitor.serialize(message.nodes)), + new xml.CR(6)); + + unit.children.push(new xml.CR(6), segment, new xml.CR(4)); + + units.push(new xml.CR(4), unit); + }); + + const file = + new xml.Tag('file', {'original': 'ng.template', id: 'ngi18n'}, [...units, new xml.CR(2)]); + + const xliff = new xml.Tag( + _XLIFF_TAG, {version: _VERSION, xmlns: _XMLNS, srcLang: locale || _DEFAULT_SOURCE_LANG}, + [new xml.CR(2), file, new xml.CR()]); + + return xml.serialize([ + new xml.Declaration({version: '1.0', encoding: 'UTF-8'}), new xml.CR(), xliff, new xml.CR() + ]); + } + + load(content: string, url: string): + {locale: string, i18nNodesByMsgId: {[msgId: string]: i18n.Node[]}} { + // xliff to xml nodes + const xliff2Parser = new Xliff2Parser(); + const {locale, msgIdToHtml, errors} = xliff2Parser.parse(content, url); + + // xml nodes to i18n nodes + const i18nNodesByMsgId: {[msgId: string]: i18n.Node[]} = {}; + const converter = new XmlToI18n(); + + Object.keys(msgIdToHtml).forEach(msgId => { + const {i18nNodes, errors: e} = converter.convert(msgIdToHtml[msgId], url); + errors.push(...e); + i18nNodesByMsgId[msgId] = i18nNodes; + }); + + if (errors.length) { + throw new Error(`xliff2 parse errors:\n${errors.join('\n')}`); + } + + return {locale, i18nNodesByMsgId}; + } + + digest(message: i18n.Message): string { return decimalDigest(message); } +} + +class _WriteVisitor implements i18n.Visitor { + private _nextPlaceholderId: number; + + visitText(text: i18n.Text, context?: any): xml.Node[] { return [new xml.Text(text.value)]; } + + visitContainer(container: i18n.Container, context?: any): xml.Node[] { + const nodes: xml.Node[] = []; + container.children.forEach((node: i18n.Node) => nodes.push(...node.visit(this))); + return nodes; + } + + visitIcu(icu: i18n.Icu, context?: any): xml.Node[] { + const nodes = [new xml.Text(`{${icu.expressionPlaceholder}, ${icu.type}, `)]; + + Object.keys(icu.cases).forEach((c: string) => { + nodes.push(new xml.Text(`${c} {`), ...icu.cases[c].visit(this), new xml.Text(`} `)); + }); + + nodes.push(new xml.Text(`}`)); + + return nodes; + } + + visitTagPlaceholder(ph: i18n.TagPlaceholder, context?: any): xml.Node[] { + const type = getTypeForTag(ph.tag); + + if (ph.isVoid) { + const tagPh = new xml.Tag(_PLACEHOLDER_TAG, { + id: (this._nextPlaceholderId++).toString(), + equiv: ph.startName, + type: type, + disp: `<${ph.tag}/>`, + }); + return [tagPh]; + } + + const tagPc = new xml.Tag(_PLACEHOLDER_SPANNING_TAG, { + id: (this._nextPlaceholderId++).toString(), + equivStart: ph.startName, + equivEnd: ph.closeName, + type: type, + dispStart: `<${ph.tag}>`, + dispEnd: ``, + }); + const nodes: xml.Node[] = [].concat(...ph.children.map(node => node.visit(this))); + if (nodes.length) { + nodes.forEach((node: xml.Node) => tagPc.children.push(node)); + } else { + tagPc.children.push(new xml.Text('')); + } + + return [tagPc]; + } + + visitPlaceholder(ph: i18n.Placeholder, context?: any): xml.Node[] { + return [new xml.Tag(_PLACEHOLDER_TAG, { + id: (this._nextPlaceholderId++).toString(), + equiv: ph.name, + disp: `{{${ph.value}}}`, + })]; + } + + visitIcuPlaceholder(ph: i18n.IcuPlaceholder, context?: any): xml.Node[] { + return [new xml.Tag(_PLACEHOLDER_TAG, {id: (this._nextPlaceholderId++).toString()})]; + } + + serialize(nodes: i18n.Node[]): xml.Node[] { + this._nextPlaceholderId = 0; + return [].concat(...nodes.map(node => node.visit(this))); + } +} + +// Extract messages as xml nodes from the xliff file +class Xliff2Parser implements ml.Visitor { + private _unitMlString: string; + private _errors: I18nError[]; + private _msgIdToHtml: {[msgId: string]: string}; + private _locale: string|null = null; + + parse(xliff: string, url: string) { + this._unitMlString = null; + this._msgIdToHtml = {}; + + const xml = new XmlParser().parse(xliff, url, false); + + this._errors = xml.errors; + ml.visitAll(this, xml.rootNodes, null); + + return { + msgIdToHtml: this._msgIdToHtml, + errors: this._errors, + locale: this._locale, + }; + } + + visitElement(element: ml.Element, context: any): any { + switch (element.name) { + case _UNIT_TAG: + this._unitMlString = null; + const idAttr = element.attrs.find((attr) => attr.name === 'id'); + if (!idAttr) { + this._addError(element, `<${_UNIT_TAG}> misses the "id" attribute`); + } else { + const id = idAttr.value; + if (this._msgIdToHtml.hasOwnProperty(id)) { + this._addError(element, `Duplicated translations for msg ${id}`); + } else { + ml.visitAll(this, element.children, null); + if (typeof this._unitMlString === 'string') { + this._msgIdToHtml[id] = this._unitMlString; + } else { + this._addError(element, `Message ${id} misses a translation`); + } + } + } + break; + + case _SOURCE_TAG: + // ignore source message + break; + + case _TARGET_TAG: + const innerTextStart = element.startSourceSpan.end.offset; + const innerTextEnd = element.endSourceSpan.start.offset; + const content = element.startSourceSpan.start.file.content; + const innerText = content.slice(innerTextStart, innerTextEnd); + this._unitMlString = innerText; + break; + + case _XLIFF_TAG: + const localeAttr = element.attrs.find((attr) => attr.name === 'trgLang'); + if (localeAttr) { + this._locale = localeAttr.value; + } + + const versionAttr = element.attrs.find((attr) => attr.name === 'version'); + if (versionAttr) { + const version = versionAttr.value; + if (version !== '2.0') { + this._addError( + element, + `The XLIFF file version ${version} is not compatible with XLIFF 2.0 serializer`); + } else { + ml.visitAll(this, element.children, null); + } + } + break; + default: + ml.visitAll(this, element.children, null); + } + } + + visitAttribute(attribute: ml.Attribute, context: any): any {} + + visitText(text: ml.Text, context: any): any {} + + visitComment(comment: ml.Comment, context: any): any {} + + visitExpansion(expansion: ml.Expansion, context: any): any {} + + visitExpansionCase(expansionCase: ml.ExpansionCase, context: any): any {} + + private _addError(node: ml.Node, message: string): void { + this._errors.push(new I18nError(node.sourceSpan, message)); + } +} + +// Convert ml nodes (xliff syntax) to i18n nodes +class XmlToI18n implements ml.Visitor { + private _errors: I18nError[]; + + convert(message: string, url: string) { + const xmlIcu = new XmlParser().parse(message, url, true); + this._errors = xmlIcu.errors; + + const i18nNodes = this._errors.length > 0 || xmlIcu.rootNodes.length == 0 ? + [] : + [].concat(...ml.visitAll(this, xmlIcu.rootNodes)); + + return { + i18nNodes, + errors: this._errors, + }; + } + + visitText(text: ml.Text, context: any) { return new i18n.Text(text.value, text.sourceSpan); } + + visitElement(el: ml.Element, context: any): i18n.Node[] { + switch (el.name) { + case _PLACEHOLDER_TAG: + const nameAttr = el.attrs.find((attr) => attr.name === 'equiv'); + if (nameAttr) { + return [new i18n.Placeholder('', nameAttr.value, el.sourceSpan)]; + } + + this._addError(el, `<${_PLACEHOLDER_TAG}> misses the "equiv" attribute`); + break; + case _PLACEHOLDER_SPANNING_TAG: + const startAttr = el.attrs.find((attr) => attr.name === 'equivStart'); + const endAttr = el.attrs.find((attr) => attr.name === 'equivEnd'); + + if (!startAttr) { + this._addError(el, `<${_PLACEHOLDER_TAG}> misses the "equivStart" attribute`); + } else if (!endAttr) { + this._addError(el, `<${_PLACEHOLDER_TAG}> misses the "equivEnd" attribute`); + } else { + const startId = startAttr.value; + const endId = endAttr.value; + + return [].concat( + new i18n.Placeholder('', startId, el.sourceSpan), + ...el.children.map(node => node.visit(this, null)), + new i18n.Placeholder('', endId, el.sourceSpan)); + } + break; + default: + this._addError(el, `Unexpected tag`); + } + } + + visitExpansion(icu: ml.Expansion, context: any) { + const caseMap: {[value: string]: i18n.Node} = {}; + + ml.visitAll(this, icu.cases).forEach((c: any) => { + caseMap[c.value] = new i18n.Container(c.nodes, icu.sourceSpan); + }); + + return new i18n.Icu(icu.switchValue, icu.type, caseMap, icu.sourceSpan); + } + + visitExpansionCase(icuCase: ml.ExpansionCase, context: any): any { + return { + value: icuCase.value, + nodes: [].concat(...ml.visitAll(this, icuCase.expression)), + }; + } + + visitComment(comment: ml.Comment, context: any) {} + + visitAttribute(attribute: ml.Attribute, context: any) {} + + private _addError(node: ml.Node, message: string): void { + this._errors.push(new I18nError(node.sourceSpan, message)); + } +} + +function getTypeForTag(tag: string): string { + switch (tag.toLowerCase()) { + case 'br': + case 'b': + case 'i': + case 'u': + return 'fmt'; + case 'img': + return 'image'; + case 'a': + return 'link'; + default: + return 'other'; + } +} \ No newline at end of file diff --git a/packages/compiler/test/i18n/serializers/xliff2_spec.ts b/packages/compiler/test/i18n/serializers/xliff2_spec.ts new file mode 100644 index 0000000000..d20548c48d --- /dev/null +++ b/packages/compiler/test/i18n/serializers/xliff2_spec.ts @@ -0,0 +1,336 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {escapeRegExp} from '@angular/compiler/src/util'; + +import {serializeNodes} from '../../../src/i18n/digest'; +import {MessageBundle} from '../../../src/i18n/message_bundle'; +import {Xliff2} from '../../../src/i18n/serializers/xliff2'; +import {HtmlParser} from '../../../src/ml_parser/html_parser'; +import {DEFAULT_INTERPOLATION_CONFIG} from '../../../src/ml_parser/interpolation_config'; + +const HTML = ` +

not translatable

+

translatable element with placeholders {{ interpolation}}

+{ count, plural, =0 {

test

}} +

foo

+

{{interpolation}} Text

+


+

hello

+

{ count, plural, =0 { { sex, select, other {

deeply nested

}} }}

+

{ count, plural, =0 { { sex, select, other {

deeply nested

}} }}

+`; + +const WRITE_XLIFF = ` + + + + + translatable attribute + + + + + translatable element with placeholders + + + + + {VAR_PLURAL, plural, =0 {test} } + + + + + d + m + + + foo + + + + + nested + + + Text + + + + + ph names + + + + + + + + empty element + + + hello + + + + + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {deeply nested} } } } + + + + + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {deeply nested} } } } + + + + +`; + +const LOAD_XLIFF = ` + + + + + translatable attribute + etubirtta elbatalsnart + + + + + translatable element with placeholders + sredlohecalp htiw tnemele elbatalsnart + + + + + {VAR_PLURAL, plural, =0 {test} } + {VAR_PLURAL, plural, =0 {TEST} } + + + + + d + m + + + foo + oof + + + + + nested + + + Text + txeT + + + + + ph names + + + + + + + + + empty element + + + hello + olleh + + + + + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {deeply nested} } } } + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {profondément imbriqué} } } } + + + + + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {deeply nested} } } } + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {profondément imbriqué} } } } + + + + +`; + +export function main(): void { + const serializer = new Xliff2(); + + function toXliff(html: string, locale: string | null = null): string { + const catalog = new MessageBundle(new HtmlParser, [], {}, locale); + catalog.updateFromTemplate(html, '', DEFAULT_INTERPOLATION_CONFIG); + return catalog.write(serializer); + } + + function loadAsMap(xliff: string): {[id: string]: string} { + const {i18nNodesByMsgId} = serializer.load(xliff, 'url'); + + const msgMap: {[id: string]: string} = {}; + Object.keys(i18nNodesByMsgId) + .forEach(id => msgMap[id] = serializeNodes(i18nNodesByMsgId[id]).join('')); + + return msgMap; + } + + describe('XLIFF 2.0 serializer', () => { + describe('write', () => { + it('should write a valid xliff 2.0 file', + () => { expect(toXliff(HTML)).toEqual(WRITE_XLIFF); }); + it('should write a valid xliff 2.0 file with a source language', + () => { expect(toXliff(HTML, 'fr')).toContain('srcLang="fr"'); }); + }); + + describe('load', () => { + it('should load XLIFF files', () => { + expect(loadAsMap(LOAD_XLIFF)).toEqual({ + '1933478729560469763': 'etubirtta elbatalsnart', + '7056919470098446707': + ' sredlohecalp htiw tnemele elbatalsnart', + '2981514368455622387': + '{VAR_PLURAL, plural, =0 {[, TEST, ]}}', + 'i': 'oof', + '6440235004920703622': + 'txeT ', + '8779402634269838862': + '', + '6536355551500405293': ' olleh', + 'baz': + '{VAR_PLURAL, plural, =0 {[{VAR_SELECT, select, other {[, profondément imbriqué, ]}}, ]}}', + '2015957479576096115': + '{VAR_PLURAL, plural, =0 {[{VAR_SELECT, select, other {[, profondément imbriqué, ]}}, ]}}' + }); + }); + + it('should return the target locale', + () => { expect(serializer.load(LOAD_XLIFF, 'url').locale).toEqual('fr'); }); + }); + + describe('structure errors', () => { + it('should throw when a wrong xliff version is used', () => { + const XLIFF = ` + + + + + + + + + +`; + + expect(() => { + loadAsMap(XLIFF); + }).toThrowError(/The XLIFF file version 1.2 is not compatible with XLIFF 2.0 serializer/); + }); + + it('should throw when an unit has no translation', () => { + const XLIFF = ` + + + + + + + + +`; + + expect(() => { + loadAsMap(XLIFF); + }).toThrowError(/Message missingtarget misses a translation/); + }); + + + it('should throw when an unit has no id attribute', () => { + const XLIFF = ` + + + + + + + + + +`; + + expect(() => { loadAsMap(XLIFF); }).toThrowError(/ misses the "id" attribute/); + }); + + it('should throw on duplicate unit id', () => { + const XLIFF = ` + + + + + + + + + + + + + + + +`; + + expect(() => { + loadAsMap(XLIFF); + }).toThrowError(/Duplicated translations for msg deadbeef/); + }); + }); + + describe('message errors', () => { + it('should throw on unknown message tags', () => { + const XLIFF = ` + + + + + + msg should contain only ph and pc tags + + + +`; + + expect(() => { loadAsMap(XLIFF); }) + .toThrowError(new RegExp( + escapeRegExp(`[ERROR ->]msg should contain only ph and pc tags`))); + }); + + it('should throw when a placeholder misses an id attribute', () => { + const XLIFF = ` + + + + + + + + + +`; + + expect(() => { + loadAsMap(XLIFF); + }).toThrowError(new RegExp(escapeRegExp(` misses the "equiv" attribute`))); + }); + }); + }); +} \ No newline at end of file diff --git a/scripts/ci/offline_compiler_test.sh b/scripts/ci/offline_compiler_test.sh index 810c3bbdc7..665fa405fc 100755 --- a/scripts/ci/offline_compiler_test.sh +++ b/scripts/ci/offline_compiler_test.sh @@ -58,6 +58,7 @@ cp -v package.json $TMP ./node_modules/.bin/ngc -p tsconfig-build.json --i18nFile=src/messages.fi.xlf --locale=fi --i18nFormat=xlf ./node_modules/.bin/ng-xi18n -p tsconfig-xi18n.json --i18nFormat=xlf --locale=fr + ./node_modules/.bin/ng-xi18n -p tsconfig-xi18n.json --i18nFormat=xlf2 --outFile=messages.xliff2.xlf ./node_modules/.bin/ng-xi18n -p tsconfig-xi18n.json --i18nFormat=xmb --outFile=custom_file.xmb # Removed until #15219 is fixed