From 2c623fde16dcd5958762dba1a752995d1459e45b Mon Sep 17 00:00:00 2001 From: Pete Bacon Darwin Date: Mon, 28 Oct 2019 12:51:45 +0000 Subject: [PATCH] feat(ivy): i18n - support inlining of XTB formatted translation files (#33444) This commit implements the `XtbTranslationParser`, which can read XTB formatted files. PR Close #33444 --- .../localize/src/tools/src/translate/main.ts | 2 + .../translation_parsers/translation_utils.ts | 3 + .../xtb_translation_parser.ts | 103 +++++++ .../integration/locales/messages.it.xtb | 13 + .../test/translate/integration/main_spec.ts | 26 +- .../xtb_translation_parser_spec.ts | 281 ++++++++++++++++++ 6 files changed, 424 insertions(+), 4 deletions(-) create mode 100644 packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts create mode 100644 packages/localize/src/tools/test/translate/integration/locales/messages.it.xtb create mode 100644 packages/localize/src/tools/test/translate/translation_files/translation_parsers/xtb_translation_parser_spec.ts diff --git a/packages/localize/src/tools/src/translate/main.ts b/packages/localize/src/tools/src/translate/main.ts index 1c075a9db74b4..3c2c04233d639 100644 --- a/packages/localize/src/tools/src/translate/main.ts +++ b/packages/localize/src/tools/src/translate/main.ts @@ -18,6 +18,7 @@ import {TranslationLoader} from './translation_files/translation_loader'; import {SimpleJsonTranslationParser} from './translation_files/translation_parsers/simple_json_translation_parser'; import {Xliff1TranslationParser} from './translation_files/translation_parsers/xliff1_translation_parser'; import {Xliff2TranslationParser} from './translation_files/translation_parsers/xliff2_translation_parser'; +import {XtbTranslationParser} from './translation_files/translation_parsers/xtb_translation_parser'; import {Translator} from './translator'; import {Diagnostics} from '../diagnostics'; @@ -141,6 +142,7 @@ export function translateFiles({sourceRootPath, sourceFilePaths, translationFile [ new Xliff2TranslationParser(), new Xliff1TranslationParser(), + new XtbTranslationParser(diagnostics), new SimpleJsonTranslationParser(), ], diagnostics); diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/translation_utils.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/translation_utils.ts index 5daafe701b981..92111e458be1a 100644 --- a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/translation_utils.ts +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/translation_utils.ts @@ -27,6 +27,9 @@ export function parseInnerRange(element: Element): Node[] { const xml = xmlParser.parse( element.sourceSpan.start.file.content, element.sourceSpan.start.file.url, {tokenizeExpansionForms: true, range: getInnerRange(element)}); + if (xml.errors.length) { + throw xml.errors.map(e => new TranslationParseError(e.span, e.msg).toString()).join('\n'); + } return xml.rootNodes; } diff --git a/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts new file mode 100644 index 0000000000000..a0358f3491181 --- /dev/null +++ b/packages/localize/src/tools/src/translate/translation_files/translation_parsers/xtb_translation_parser.ts @@ -0,0 +1,103 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {Element, Node, XmlParser, visitAll} from '@angular/compiler'; +import {ɵParsedTranslation} from '@angular/localize'; +import {extname} from 'path'; + +import {Diagnostics} from '../../../diagnostics'; +import {BaseVisitor} from '../base_visitor'; +import {MessageSerializer} from '../message_serialization/message_serializer'; +import {TargetMessageRenderer} from '../message_serialization/target_message_renderer'; + +import {TranslationParseError} from './translation_parse_error'; +import {ParsedTranslationBundle, TranslationParser} from './translation_parser'; +import {getAttrOrThrow, parseInnerRange} from './translation_utils'; + + + +/** + * A translation parser that can load XB files. + */ +export class XtbTranslationParser implements TranslationParser { + constructor(private diagnostics: Diagnostics) {} + + canParse(filePath: string, contents: string): boolean { + const extension = extname(filePath); + return (extension === '.xtb' || extension === '.xmb') && + contents.includes(' elements can not be nested'); + } + const langAttr = element.attrs.find((attr) => attr.name === 'lang'); + bundle = {locale: langAttr && langAttr.value, translations: {}}; + visitAll(this, element.children, bundle); + return bundle; + + case 'translation': + if (!bundle) { + throw new TranslationParseError( + element.sourceSpan, ' must be inside a '); + } + const id = getAttrOrThrow(element, 'id'); + if (bundle.translations.hasOwnProperty(id)) { + throw new TranslationParseError( + element.sourceSpan, `Duplicated translations for message "${id}"`); + } else { + try { + bundle.translations[id] = serializeTargetMessage(element); + } catch (error) { + if (typeof error === 'string') { + this.diagnostics.warn( + `Could not parse message with id "${id}" - perhaps it has an unrecognised ICU format?\n` + + error); + } else { + throw error; + } + } + } + break; + + default: + throw new TranslationParseError(element.sourceSpan, 'Unexpected tag'); + } + } +} + +function serializeTargetMessage(source: Element): ɵParsedTranslation { + const serializer = new MessageSerializer( + new TargetMessageRenderer(), + {inlineElements: [], placeholder: {elementName: 'ph', nameAttribute: 'name'}}); + return serializer.serialize(parseInnerRange(source)); +} diff --git a/packages/localize/src/tools/test/translate/integration/locales/messages.it.xtb b/packages/localize/src/tools/test/translate/integration/locales/messages.it.xtb new file mode 100644 index 0000000000000..8adb3f36c4584 --- /dev/null +++ b/packages/localize/src/tools/test/translate/integration/locales/messages.it.xtb @@ -0,0 +1,13 @@ + + + + + + + + + +]> + + Ciao, ! + \ No newline at end of file diff --git a/packages/localize/src/tools/test/translate/integration/main_spec.ts b/packages/localize/src/tools/test/translate/integration/main_spec.ts index 8fdb9bdbc2bff..6f15de78075ad 100644 --- a/packages/localize/src/tools/test/translate/integration/main_spec.ts +++ b/packages/localize/src/tools/test/translate/integration/main_spec.ts @@ -29,7 +29,8 @@ describe('translateFiles()', () => { sourceFilePaths: resolveAll(__dirname + '/test_files', ['test-1.txt', 'test-2.txt']), outputPathFn, translationFilePaths: resolveAll( - __dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']), + __dirname + '/locales', + ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']), translationFileLocales: [], diagnostics, missingTranslation: 'error' }); @@ -48,6 +49,10 @@ describe('translateFiles()', () => { .toEqual('Contents of test-1.txt'); expect(FileUtils.readFile(resolve(testDir, 'es', 'test-2.txt'))) .toEqual('Contents of test-2.txt'); + expect(FileUtils.readFile(resolve(testDir, 'it', 'test-1.txt'))) + .toEqual('Contents of test-1.txt'); + expect(FileUtils.readFile(resolve(testDir, 'it', 'test-2.txt'))) + .toEqual('Contents of test-2.txt'); }); it('should translate and copy source-code files to the destination folders', () => { @@ -57,7 +62,8 @@ describe('translateFiles()', () => { sourceRootPath: resolve(__dirname, 'test_files'), sourceFilePaths: resolveAll(__dirname + '/test_files', ['test.js']), outputPathFn, translationFilePaths: resolveAll( - __dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']), + __dirname + '/locales', + ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']), translationFileLocales: [], diagnostics, missingTranslation: 'error', }); @@ -70,6 +76,8 @@ describe('translateFiles()', () => { .toEqual(`var name="World";var message="Guten Tag, "+name+"!";`); expect(FileUtils.readFile(resolve(testDir, 'es', 'test.js'))) .toEqual(`var name="World";var message="Hola, "+name+"!";`); + expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js'))) + .toEqual(`var name="World";var message="Ciao, "+name+"!";`); }); it('should translate and copy source-code files overriding the locales', () => { @@ -79,7 +87,8 @@ describe('translateFiles()', () => { sourceRootPath: resolve(__dirname, 'test_files'), sourceFilePaths: resolveAll(__dirname + '/test_files', ['test.js']), outputPathFn, translationFilePaths: resolveAll( - __dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']), + __dirname + '/locales', + ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']), translationFileLocales: ['xde', undefined, 'fr'], diagnostics, missingTranslation: 'error', }); @@ -97,6 +106,8 @@ describe('translateFiles()', () => { .toEqual(`var name="World";var message="Hola, "+name+"!";`); expect(FileUtils.readFile(resolve(testDir, 'fr', 'test.js'))) .toEqual(`var name="World";var message="Bonjour, "+name+"!";`); + expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js'))) + .toEqual(`var name="World";var message="Ciao, "+name+"!";`); }); it('should transform and/or copy files to the destination folders', () => { @@ -108,7 +119,8 @@ describe('translateFiles()', () => { resolveAll(__dirname + '/test_files', ['test-1.txt', 'test-2.txt', 'test.js']), outputPathFn, translationFilePaths: resolveAll( - __dirname + '/locales', ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf']), + __dirname + '/locales', + ['messages.de.json', 'messages.es.xlf', 'messages.fr.xlf', 'messages.it.xtb']), translationFileLocales: [], diagnostics, missingTranslation: 'error', }); @@ -127,6 +139,10 @@ describe('translateFiles()', () => { .toEqual('Contents of test-1.txt'); expect(FileUtils.readFile(resolve(testDir, 'es', 'test-2.txt'))) .toEqual('Contents of test-2.txt'); + expect(FileUtils.readFile(resolve(testDir, 'it', 'test-1.txt'))) + .toEqual('Contents of test-1.txt'); + expect(FileUtils.readFile(resolve(testDir, 'it', 'test-2.txt'))) + .toEqual('Contents of test-2.txt'); expect(FileUtils.readFile(resolve(testDir, 'fr', 'test.js'))) .toEqual(`var name="World";var message="Bonjour, "+name+"!";`); @@ -134,6 +150,8 @@ describe('translateFiles()', () => { .toEqual(`var name="World";var message="Guten Tag, "+name+"!";`); expect(FileUtils.readFile(resolve(testDir, 'es', 'test.js'))) .toEqual(`var name="World";var message="Hola, "+name+"!";`); + expect(FileUtils.readFile(resolve(testDir, 'it', 'test.js'))) + .toEqual(`var name="World";var message="Ciao, "+name+"!";`); }); }); diff --git a/packages/localize/src/tools/test/translate/translation_files/translation_parsers/xtb_translation_parser_spec.ts b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/xtb_translation_parser_spec.ts new file mode 100644 index 0000000000000..d07babbb9b1f1 --- /dev/null +++ b/packages/localize/src/tools/test/translate/translation_files/translation_parsers/xtb_translation_parser_spec.ts @@ -0,0 +1,281 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ +import {ɵcomputeMsgId, ɵmakeParsedTranslation} from '@angular/localize'; +import {Diagnostics} from '../../../../src/diagnostics'; +import {XtbTranslationParser} from '../../../../src/translate/translation_files/translation_parsers/xtb_translation_parser'; + +describe('XtbTranslationParser', () => { + describe('canParse()', () => { + it('should return true if the file extension is `.xtb` or `.xmb` and it contains the `` tag', + () => { + const parser = new XtbTranslationParser(new Diagnostics()); + expect(parser.canParse('/some/file.xtb', '')).toBe(true); + expect(parser.canParse('/some/file.xmb', '')).toBe(true); + expect(parser.canParse('/some/file.xtb', '')).toBe(true); + expect(parser.canParse('/some/file.xmb', '')).toBe(true); + expect(parser.canParse('/some/file.json', '')).toBe(false); + expect(parser.canParse('/some/file.xmb', '')).toBe(false); + expect(parser.canParse('/some/file.xtb', '')).toBe(false); + }); + }); + + describe('parse()', () => { + it('should extract the locale from the file contents', () => { + const XTB = ` + + rab + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + expect(result.locale).toEqual('fr'); + }); + + it('should extract basic messages', () => { + const XTB = ` + + + + + + + + + ]> + + rab + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations['8841459487341224498']).toEqual(ɵmakeParsedTranslation(['rab'])); + }); + + it('should extract translations with simple placeholders', () => { + const XTB = ` + + rab + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations['8877975308926375834']) + .toEqual(ɵmakeParsedTranslation(['', 'rab', ''], ['START_PARAGRAPH', 'CLOSE_PARAGRAPH'])); + }); + + it('should extract translations with simple ICU expressions', () => { + const XTB = ` + + ** + {VAR_PLURAL, plural, =1 {rab}} + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations['7717087045075616176']) + .toEqual(ɵmakeParsedTranslation(['*', '*'], ['ICU'])); + expect(result.translations['5115002811911870583']) + .toEqual(ɵmakeParsedTranslation( + ['{VAR_PLURAL, plural, =1 {{START_PARAGRAPH}rab{CLOSE_PARAGRAPH}}}'], [])); + }); + + it('should extract translations with duplicate source messages', () => { + const XTB = ` + + oof + toto + tata + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations[ɵcomputeMsgId('foo')]).toEqual(ɵmakeParsedTranslation(['oof'])); + expect(result.translations['i']).toEqual(ɵmakeParsedTranslation(['toto'])); + expect(result.translations['bar']).toEqual(ɵmakeParsedTranslation(['tata'])); + }); + + it('should extract translations with only placeholders, which are re-ordered', () => { + const XTB = ` + + + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations[ɵcomputeMsgId('{$LINE_BREAK}{$TAG_IMG}{$TAG_IMG_1}')]) + .toEqual( + ɵmakeParsedTranslation(['', '', '', ''], ['TAG_IMG_1', 'TAG_IMG', 'LINE_BREAK'])); + }); + + it('should extract translations with empty target', () => { + /** + * Source HTML: + * + * ``` + *
hello
+ * ``` + */ + const XTB = ` + + + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations[ɵcomputeMsgId('hello {$START_TAG_SPAN}{$CLOSE_TAG_SPAN}')]) + .toEqual(ɵmakeParsedTranslation([''])); + }); + + it('should extract translations with deeply nested ICUs', () => { + /** + * Source HTML: + * + * ``` + * Test: { count, plural, =0 { { sex, select, other {

deeply nested

}} } =other {a lot}} + * ``` + * + * Note that the message gets split into two translation units: + * * The first one contains the outer message with an `ICU` placeholder + * * The second one is the ICU expansion itself + * + * Note that special markers `VAR_PLURAL` and `VAR_SELECT` are added, which are then + replaced by IVY at runtime with the actual values being rendered by the ICU expansion. + */ + const XTB = ` + + Le test: + {VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {profondément imbriqué}}} =other {beaucoup}} + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations[ɵcomputeMsgId('Test: {$ICU}')]) + .toEqual(ɵmakeParsedTranslation(['Le test: ', ''], ['ICU'])); + + expect( + result.translations[ɵcomputeMsgId( + '{VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {{START_PARAGRAPH}deeply nested{CLOSE_PARAGRAPH}}}} =other {beaucoup}}')]) + .toEqual(ɵmakeParsedTranslation([ + '{VAR_PLURAL, plural, =0 {{VAR_SELECT, select, other {{START_PARAGRAPH}profondément imbriqué{CLOSE_PARAGRAPH}}}} =other {beaucoup}}' + ])); + }); + + it('should extract translations containing multiple lines', () => { + /** + * Source HTML: + * + * ``` + *
multi + * lines
+ * ``` + */ + const XTB = ` + + multi\nlignes + `; + const parser = new XtbTranslationParser(new Diagnostics()); + const result = parser.parse('/some/file.xtb', XTB); + + expect(result.translations[ɵcomputeMsgId('multi\nlines')]) + .toEqual(ɵmakeParsedTranslation(['multi\nlignes'])); + }); + + it('should warn on unrecognised ICU messages', () => { + // See https://github.com/angular/angular/issues/14046 + + const XTB = ` + + This is a valid message + {REGION_COUNT_1, plural, =0 {unused plural form} =1 {1 region} other {{REGION_COUNT_2} regions}} + `; + + // Parsing the file should not fail + const diagnostics = new Diagnostics(); + const parser = new XtbTranslationParser(diagnostics); + const result = parser.parse('/some/file.xtb', XTB); + + // We should be able to read the valid message + expect(result.translations['valid']) + .toEqual(ɵmakeParsedTranslation(['This is a valid message'])); + + // Trying to access the invalid message should fail + expect(result.translations['invalid']).toBeUndefined(); + expect(diagnostics.messages).toContain({ + type: 'warning', + message: + `Could not parse message with id "invalid" - perhaps it has an unrecognised ICU format?\n` + + `Error: Unexpected character "EOF" (Do you have an unescaped "{" in your template? Use "{{ '{' }}") to escape it.)\n` + + `Error: Invalid ICU message. Missing '}'.` + }); + }); + + describe('[structure errors]', () => { + it('should throw when there are nested translationbundle tags', () => { + const XTB = + ''; + + expect(() => { + const parser = new XtbTranslationParser(new Diagnostics()); + parser.parse('/some/file.xtb', XTB); + }).toThrowError(/ elements can not be nested/); + }); + + it('should throw when a translation has no id attribute', () => { + const XTB = ` + + + `; + + expect(() => { + const parser = new XtbTranslationParser(new Diagnostics()); + parser.parse('/some/file.xtb', XTB); + }).toThrowError(/Missing required "id" attribute/); + }); + + it('should throw on duplicate translation id', () => { + const XTB = ` + + + + `; + + expect(() => { + const parser = new XtbTranslationParser(new Diagnostics()); + parser.parse('/some/file.xtb', XTB); + }).toThrowError(/Duplicated translations for message "deadbeef"/); + }); + }); + + describe('[message errors]', () => { + it('should throw on unknown message tags', () => { + const XTB = ` + + + + + `; + + expect(() => { + const parser = new XtbTranslationParser(new Diagnostics()); + parser.parse('/some/file.xtb', XTB); + }).toThrowError(/Invalid element found in message/); + }); + + it('should throw when a placeholder misses a name attribute', () => { + const XTB = ` + + + `; + + expect(() => { + const parser = new XtbTranslationParser(new Diagnostics()); + parser.parse('/some/file.xtb', XTB); + }).toThrowError(/required "name" attribute/gi); + }); + }); + }); +});